diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ColumnFamilyDescriptor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ColumnFamilyDescriptor.java index 369b2be8ecda..ea8d81043694 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ColumnFamilyDescriptor.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ColumnFamilyDescriptor.java @@ -114,6 +114,9 @@ public interface ColumnFamilyDescriptor { /** Returns Return the raw crypto key attribute for the family, or null if not set */ byte[] getEncryptionKey(); + /** Returns the encryption key namespace for this family */ + String getEncryptionKeyNamespace(); + /** Returns Return the encryption algorithm in use by this family */ String getEncryptionType(); diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ColumnFamilyDescriptorBuilder.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ColumnFamilyDescriptorBuilder.java index 42f25fdc56f4..12bb73565078 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ColumnFamilyDescriptorBuilder.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ColumnFamilyDescriptorBuilder.java @@ -167,6 +167,10 @@ public class ColumnFamilyDescriptorBuilder { @InterfaceAudience.Private public static final String ENCRYPTION_KEY = "ENCRYPTION_KEY"; private static final Bytes ENCRYPTION_KEY_BYTES = new Bytes(Bytes.toBytes(ENCRYPTION_KEY)); + @InterfaceAudience.Private + public static final String ENCRYPTION_KEY_NAMESPACE = "ENCRYPTION_KEY_NAMESPACE"; + private static final Bytes ENCRYPTION_KEY_NAMESPACE_BYTES = + new Bytes(Bytes.toBytes(ENCRYPTION_KEY_NAMESPACE)); private static final boolean DEFAULT_MOB = false; @InterfaceAudience.Private @@ -320,6 +324,7 @@ public static Map getDefaultValues() { DEFAULT_VALUES.keySet().forEach(s -> RESERVED_KEYWORDS.add(new Bytes(Bytes.toBytes(s)))); RESERVED_KEYWORDS.add(new Bytes(Bytes.toBytes(ENCRYPTION))); RESERVED_KEYWORDS.add(new Bytes(Bytes.toBytes(ENCRYPTION_KEY))); + RESERVED_KEYWORDS.add(new Bytes(Bytes.toBytes(ENCRYPTION_KEY_NAMESPACE))); RESERVED_KEYWORDS.add(new Bytes(Bytes.toBytes(IS_MOB))); RESERVED_KEYWORDS.add(new Bytes(Bytes.toBytes(MOB_THRESHOLD))); RESERVED_KEYWORDS.add(new Bytes(Bytes.toBytes(MOB_COMPACT_PARTITION_POLICY))); @@ -522,6 +527,11 @@ public ColumnFamilyDescriptorBuilder setEncryptionKey(final byte[] value) { return this; } + public ColumnFamilyDescriptorBuilder setEncryptionKeyNamespace(final String value) { + desc.setEncryptionKeyNamespace(value); + return this; + } + public ColumnFamilyDescriptorBuilder setEncryptionType(String value) { desc.setEncryptionType(value); return this; @@ -1337,6 +1347,20 @@ public ModifyableColumnFamilyDescriptor setEncryptionKey(byte[] keyBytes) { return setValue(ENCRYPTION_KEY_BYTES, new Bytes(keyBytes)); } + @Override + public String getEncryptionKeyNamespace() { + return getStringOrDefault(ENCRYPTION_KEY_NAMESPACE_BYTES, Function.identity(), null); + } + + /** + * Set the encryption key namespace attribute for the family + * @param keyNamespace the key namespace, or null to remove existing setting + * @return this (for chained invocation) + */ + public ModifyableColumnFamilyDescriptor setEncryptionKeyNamespace(String keyNamespace) { + return setValue(ENCRYPTION_KEY_NAMESPACE_BYTES, keyNamespace); + } + @Override public long getMobThreshold() { return getStringOrDefault(MOB_THRESHOLD_BYTES, Long::valueOf, DEFAULT_MOB_THRESHOLD); diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java index 2dca4f7e452d..73637f0cd20e 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java @@ -1288,6 +1288,14 @@ public enum OperationStatusCode { public static final String CRYPTO_KEYPROVIDER_PARAMETERS_KEY = "hbase.crypto.keyprovider.parameters"; + /** Configuration key for the managed crypto key provider, a class name */ + public static final String CRYPTO_MANAGED_KEYPROVIDER_CONF_KEY = + "hbase.crypto.managed.keyprovider"; + + /** Configuration key for the managed crypto key provider parameters */ + public static final String CRYPTO_MANAGED_KEYPROVIDER_PARAMETERS_KEY = + "hbase.crypto.managed.keyprovider.parameters"; + /** Configuration key for the name of the master key for the cluster, a string */ public static final String CRYPTO_MASTERKEY_NAME_CONF_KEY = "hbase.crypto.master.key.name"; diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Encryption.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Encryption.java index 91af77361a0e..e8d965adebba 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Encryption.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Encryption.java @@ -29,6 +29,7 @@ import java.util.Arrays; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.function.BiFunction; import javax.crypto.SecretKeyFactory; import javax.crypto.spec.PBEKeySpec; import javax.crypto.spec.SecretKeySpec; @@ -556,32 +557,45 @@ public static CipherProvider getCipherProvider(Configuration conf) { } } - static final Map, KeyProvider> keyProviderCache = new ConcurrentHashMap<>(); + static final Map, Object> keyProviderCache = new ConcurrentHashMap<>(); - public static KeyProvider getKeyProvider(Configuration conf) { - String providerClassName = - conf.get(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, KeyStoreKeyProvider.class.getName()); - String providerParameters = conf.get(HConstants.CRYPTO_KEYPROVIDER_PARAMETERS_KEY, ""); - try { - Pair providerCacheKey = new Pair<>(providerClassName, providerParameters); - KeyProvider provider = keyProviderCache.get(providerCacheKey); - if (provider != null) { - return provider; - } - provider = (KeyProvider) ReflectionUtils - .newInstance(getClassLoaderForClass(KeyProvider.class).loadClass(providerClassName), conf); - provider.init(providerParameters); - if (provider instanceof ManagedKeyProvider) { - ((ManagedKeyProvider) provider).initConfig(conf); - } - if (LOG.isDebugEnabled()) { - LOG.debug("Installed " + providerClassName + " into key provider cache"); + private static Object createProvider(final Configuration conf, String classNameKey, + String parametersKey, Class defaultProviderClass, ClassLoader classLoaderForClass, + BiFunction initFunction) { + String providerClassName = conf.get(classNameKey, defaultProviderClass.getName()); + String providerParameters = conf.get(parametersKey, ""); + Pair providerCacheKey = new Pair<>(providerClassName, providerParameters); + Object provider = keyProviderCache.get(providerCacheKey); + if (provider == null) { + try { + provider = + ReflectionUtils.newInstance(classLoaderForClass.loadClass(providerClassName), conf); + initFunction.apply(provider, providerParameters); + } catch (Exception e) { + throw new RuntimeException(e); } keyProviderCache.put(providerCacheKey, provider); - return provider; - } catch (Exception e) { - throw new RuntimeException(e); + LOG.debug("Installed " + providerClassName + " into key provider cache"); } + return provider; + } + + public static KeyProvider getKeyProvider(final Configuration conf) { + return (KeyProvider) createProvider(conf, HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, + HConstants.CRYPTO_KEYPROVIDER_PARAMETERS_KEY, KeyStoreKeyProvider.class, + getClassLoaderForClass(KeyProvider.class), (provider, providerParameters) -> { + ((KeyProvider) provider).init(providerParameters); + return null; + }); + } + + public static ManagedKeyProvider getManagedKeyProvider(final Configuration conf) { + return (ManagedKeyProvider) createProvider(conf, HConstants.CRYPTO_MANAGED_KEYPROVIDER_CONF_KEY, + HConstants.CRYPTO_MANAGED_KEYPROVIDER_PARAMETERS_KEY, ManagedKeyProvider.class, + getClassLoaderForClass(ManagedKeyProvider.class), (provider, providerParameters) -> { + ((ManagedKeyProvider) provider).initConfig(conf, providerParameters); + return null; + }); } @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.UNITTEST) diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/ManagedKeyProvider.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/ManagedKeyProvider.java index 512f78a1f9f5..c3adc5867bd1 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/ManagedKeyProvider.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/ManagedKeyProvider.java @@ -25,16 +25,16 @@ /** * Interface for key providers of managed keys. Defines methods for generating and managing managed - * keys, as well as handling key storage and retrieval. The interface extends the basic - * {@link KeyProvider} interface with additional methods for working with managed keys. + * keys, as well as handling key storage and retrieval. */ @InterfaceAudience.Public -public interface ManagedKeyProvider extends KeyProvider { +public interface ManagedKeyProvider { /** * Initialize the provider with the given configuration. - * @param conf Hadoop configuration + * @param conf Hadoop configuration + * @param providerParameters provider parameters */ - void initConfig(Configuration conf); + void initConfig(Configuration conf, String providerParameters); /** * Retrieve the system key using the given system identifier. diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/ManagedKeyStoreKeyProvider.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/ManagedKeyStoreKeyProvider.java index 74f892f7ad89..15e49bd692e4 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/ManagedKeyStoreKeyProvider.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/ManagedKeyStoreKeyProvider.java @@ -31,6 +31,7 @@ public class ManagedKeyStoreKeyProvider extends KeyStoreKeyProvider implements ManagedKeyProvider { public static final String KEY_METADATA_ALIAS = "KeyAlias"; public static final String KEY_METADATA_CUST = "KeyCustodian"; + public static final String KEY_METADATA_NAMESPACE = "KeyNamespace"; private static final java.lang.reflect.Type KEY_METADATA_TYPE = new TypeToken>() { @@ -39,8 +40,11 @@ public class ManagedKeyStoreKeyProvider extends KeyStoreKeyProvider implements M private Configuration conf; @Override - public void initConfig(Configuration conf) { + public void initConfig(Configuration conf, String providerParameters) { this.conf = conf; + if (providerParameters != null) { + super.init(providerParameters); + } } @Override @@ -56,8 +60,8 @@ public ManagedKeyData getSystemKey(byte[] clusterId) { throw new RuntimeException("Unable to find system key with alias: " + systemKeyAlias); } // Encode clusterId too for consistency with that of key custodian. - String keyMetadata = - generateKeyMetadata(systemKeyAlias, ManagedKeyProvider.encodeToStr(clusterId)); + String keyMetadata = generateKeyMetadata(systemKeyAlias, + ManagedKeyProvider.encodeToStr(clusterId), ManagedKeyData.KEY_SPACE_GLOBAL); return new ManagedKeyData(clusterId, ManagedKeyData.KEY_SPACE_GLOBAL, key, ManagedKeyState.ACTIVE, keyMetadata); } @@ -66,9 +70,25 @@ public ManagedKeyData getSystemKey(byte[] clusterId) { public ManagedKeyData getManagedKey(byte[] key_cust, String key_namespace) throws IOException { checkConfig(); String encodedCust = ManagedKeyProvider.encodeToStr(key_cust); - String aliasConfKey = - HConstants.CRYPTO_MANAGED_KEY_STORE_CONF_KEY_PREFIX + encodedCust + "." + "alias"; - String keyMetadata = generateKeyMetadata(conf.get(aliasConfKey, null), encodedCust); + + // Handle null key_namespace by defaulting to global namespace + if (key_namespace == null) { + key_namespace = ManagedKeyData.KEY_SPACE_GLOBAL; + } + + // Get alias configuration for the specific custodian+namespace combination + String aliasConfKey = buildAliasConfKey(encodedCust, key_namespace); + String alias = conf.get(aliasConfKey, null); + + // Generate metadata with actual alias (used for both success and failure cases) + String keyMetadata = generateKeyMetadata(alias, encodedCust, key_namespace); + + // If no alias is configured for this custodian+namespace combination, treat as key not found + if (alias == null) { + return new ManagedKeyData(key_cust, key_namespace, null, ManagedKeyState.FAILED, keyMetadata); + } + + // Namespaces match, proceed to get the key return unwrapKey(keyMetadata, null); } @@ -77,17 +97,21 @@ public ManagedKeyData unwrapKey(String keyMetadataStr, byte[] wrappedKey) throws Map keyMetadata = GsonUtil.getDefaultInstance().fromJson(keyMetadataStr, KEY_METADATA_TYPE); String encodedCust = keyMetadata.get(KEY_METADATA_CUST); - String activeStatusConfKey = - HConstants.CRYPTO_MANAGED_KEY_STORE_CONF_KEY_PREFIX + encodedCust + ".active"; + String namespace = keyMetadata.get(KEY_METADATA_NAMESPACE); + if (namespace == null) { + // For backwards compatibility, default to global namespace + namespace = ManagedKeyData.KEY_SPACE_GLOBAL; + } + String activeStatusConfKey = buildActiveStatusConfKey(encodedCust, namespace); boolean isActive = conf.getBoolean(activeStatusConfKey, true); byte[] key_cust = ManagedKeyProvider.decodeToBytes(encodedCust); String alias = keyMetadata.get(KEY_METADATA_ALIAS); Key key = alias != null ? getKey(alias) : null; if (key != null) { - return new ManagedKeyData(key_cust, ManagedKeyData.KEY_SPACE_GLOBAL, key, + return new ManagedKeyData(key_cust, namespace, key, isActive ? ManagedKeyState.ACTIVE : ManagedKeyState.INACTIVE, keyMetadataStr); } - return new ManagedKeyData(key_cust, ManagedKeyData.KEY_SPACE_GLOBAL, null, + return new ManagedKeyData(key_cust, namespace, null, isActive ? ManagedKeyState.FAILED : ManagedKeyState.DISABLED, keyMetadataStr); } @@ -98,9 +122,24 @@ private void checkConfig() { } public static String generateKeyMetadata(String aliasName, String encodedCust) { - Map metadata = new HashMap<>(2); + return generateKeyMetadata(aliasName, encodedCust, ManagedKeyData.KEY_SPACE_GLOBAL); + } + + public static String generateKeyMetadata(String aliasName, String encodedCust, String namespace) { + Map metadata = new HashMap<>(3); metadata.put(KEY_METADATA_ALIAS, aliasName); metadata.put(KEY_METADATA_CUST, encodedCust); + metadata.put(KEY_METADATA_NAMESPACE, namespace); return GsonUtil.getDefaultInstance().toJson(metadata, HashMap.class); } + + private String buildAliasConfKey(String encodedCust, String namespace) { + return HConstants.CRYPTO_MANAGED_KEY_STORE_CONF_KEY_PREFIX + encodedCust + "." + namespace + + ".alias"; + } + + private String buildActiveStatusConfKey(String encodedCust, String namespace) { + return HConstants.CRYPTO_MANAGED_KEY_STORE_CONF_KEY_PREFIX + encodedCust + "." + namespace + + ".active"; + } } diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/MockAesKeyProvider.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/MockAesKeyProvider.java index 39f460e062ae..0b85f1d76de1 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/MockAesKeyProvider.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/MockAesKeyProvider.java @@ -50,7 +50,11 @@ public Key[] getKeys(String[] aliases) { if (keys.containsKey(aliases[i])) { result[i] = keys.get(aliases[i]); } else { - result[i] = new SecretKeySpec(Encryption.hash128(aliases[i]), "AES"); + // When not caching keys, we want to make the key generation deterministic. + result[i] = new SecretKeySpec( + Encryption.hash128( + cacheKeys ? aliases[i] + "-" + String.valueOf(System.currentTimeMillis()) : aliases[i]), + "AES"); if (cacheKeys) { keys.put(aliases[i], result[i]); } @@ -58,4 +62,8 @@ public Key[] getKeys(String[] aliases) { } return result; } + + public void clearKeys() { + keys.clear(); + } } diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/KeymetaTestUtils.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/KeymetaTestUtils.java index 3a8fb3d32464..f02979cd9893 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/KeymetaTestUtils.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/KeymetaTestUtils.java @@ -116,6 +116,14 @@ private KeymetaTestUtils() { public static void addEntry(Configuration conf, int keyLen, KeyStore store, String alias, String custodian, boolean withPasswordOnAlias, Map cust2key, Map cust2alias, Properties passwordFileProps) throws Exception { + addEntry(conf, keyLen, store, alias, custodian, withPasswordOnAlias, cust2key, cust2alias, + passwordFileProps, ManagedKeyData.KEY_SPACE_GLOBAL); + } + + public static void addEntry(Configuration conf, int keyLen, KeyStore store, String alias, + String custodian, boolean withPasswordOnAlias, Map cust2key, + Map cust2alias, Properties passwordFileProps, String namespace) + throws Exception { Preconditions.checkArgument(keyLen == 256 || keyLen == 128, "Key length must be 256 or 128"); byte[] key = MessageDigest.getInstance(keyLen == 256 ? "SHA-256" : "MD5").digest(Bytes.toBytes(alias)); @@ -124,8 +132,18 @@ public static void addEntry(Configuration conf, int keyLen, KeyStore store, Stri store.setEntry(alias, new KeyStore.SecretKeyEntry(new SecretKeySpec(key, "AES")), new KeyStore.PasswordProtection(withPasswordOnAlias ? PASSWORD.toCharArray() : new char[0])); String encCust = Base64.getEncoder().encodeToString(custodian.getBytes()); - String confKey = HConstants.CRYPTO_MANAGED_KEY_STORE_CONF_KEY_PREFIX + encCust + "." + "alias"; - conf.set(confKey, alias); + + // Use new format: PREFIX.{encodedCust}.{namespace}.alias + // For global namespace use "*", for custom namespace use actual namespace name + String namespaceKey = ManagedKeyData.KEY_SPACE_GLOBAL.equals(namespace) ? "*" : namespace; + String aliasConfKey = + HConstants.CRYPTO_MANAGED_KEY_STORE_CONF_KEY_PREFIX + encCust + "." + namespaceKey + ".alias"; + String activeStatusConfKey = HConstants.CRYPTO_MANAGED_KEY_STORE_CONF_KEY_PREFIX + encCust + "." + + namespaceKey + ".active"; + + conf.set(aliasConfKey, alias); + conf.setBoolean(activeStatusConfKey, true); + if (passwordFileProps != null) { passwordFileProps.setProperty(alias, PASSWORD); } diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/MockManagedKeyProvider.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/MockManagedKeyProvider.java index 99c9c132d7d4..6782a7d11636 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/MockManagedKeyProvider.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/MockManagedKeyProvider.java @@ -45,8 +45,8 @@ public class MockManagedKeyProvider extends MockAesKeyProvider implements Manage private String systemKeyAlias = "default_system_key_alias"; @Override - public void initConfig(Configuration conf) { - // NO-OP + public void initConfig(Configuration conf, String providerParameters) { + super.init(providerParameters); } @Override diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/TestKeyProvider.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/TestKeyProvider.java index 741cf05744d8..14718ddfc44e 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/TestKeyProvider.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/TestKeyProvider.java @@ -55,4 +55,18 @@ public void testTestProvider() { key.getEncoded().length); } + @Test + public void testManagedKeyProvider() { + Configuration conf = HBaseConfiguration.create(); + conf.set(HConstants.CRYPTO_MANAGED_KEYPROVIDER_CONF_KEY, + MockManagedKeyProvider.class.getName()); + ManagedKeyProvider provider = Encryption.getManagedKeyProvider(conf); + assertNotNull("Null returned for managed provider", provider); + assertTrue("Provider is not the expected type", provider instanceof MockManagedKeyProvider); + + // Test that it's cached + ManagedKeyProvider provider2 = Encryption.getManagedKeyProvider(conf); + assertTrue("Provider should be cached and same instance", provider == provider2); + } + } diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/TestManagedKeyProvider.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/TestManagedKeyProvider.java index 405c5731be94..7a003f2943ed 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/TestManagedKeyProvider.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/TestManagedKeyProvider.java @@ -29,12 +29,14 @@ import java.security.KeyStore; import java.util.Arrays; import java.util.Base64; +import java.util.Collection; import java.util.HashMap; import java.util.Map; import java.util.Properties; import java.util.UUID; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseCommonTestingUtil; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.testclassification.MiscTests; @@ -57,26 +59,53 @@ public class TestManagedKeyProvider { @RunWith(Parameterized.class) @Category({ MiscTests.class, SmallTests.class }) - public static class TestManagedKeyStoreKeyProvider extends TestKeyStoreKeyProvider { + public static class TestManagedKeyStoreKeyProvider { @ClassRule public static final HBaseClassTestRule CLASS_RULE = HBaseClassTestRule.forClass(TestManagedKeyStoreKeyProvider.class); private static final String SYSTEM_KEY_ALIAS = "system-alias"; + static final HBaseCommonTestingUtil TEST_UTIL = new HBaseCommonTestingUtil(); + static byte[] KEY; + + @Parameterized.Parameter(0) + public boolean withPasswordOnAlias; + @Parameterized.Parameter(1) + public boolean withPasswordFile; + + @Parameterized.Parameters(name = "withPasswordOnAlias={0} withPasswordFile={1}") + public static Collection parameters() { + return Arrays + .asList(new Object[][] { { Boolean.TRUE, Boolean.TRUE }, { Boolean.TRUE, Boolean.FALSE }, + { Boolean.FALSE, Boolean.TRUE }, { Boolean.FALSE, Boolean.FALSE }, }); + } + + // TestManagedKeyStoreKeyProvider specific fields private Configuration conf = HBaseConfiguration.create(); private int nCustodians = 2; private ManagedKeyProvider managedKeyProvider; private Map cust2key = new HashMap<>(); private Map cust2alias = new HashMap<>(); + private Map namespaceCust2key = new HashMap<>(); + private Map namespaceCust2alias = new HashMap<>(); private String clusterId; private byte[] systemKey; @Before public void setUp() throws Exception { - super.setUp(); - managedKeyProvider = (ManagedKeyProvider) provider; - managedKeyProvider.initConfig(conf); + String providerParams = KeymetaTestUtils.setupTestKeyStore(TEST_UTIL, withPasswordOnAlias, + withPasswordFile, store -> { + Properties passwdProps = new Properties(); + try { + addCustomEntries(store, passwdProps); + } catch (Exception e) { + throw new RuntimeException(e); + } + return passwdProps; + }); + managedKeyProvider = (ManagedKeyProvider) createProvider(); + managedKeyProvider.initConfig(conf, providerParams); } protected KeyProvider createProvider() { @@ -84,7 +113,7 @@ protected KeyProvider createProvider() { } protected void addCustomEntries(KeyStore store, Properties passwdProps) throws Exception { - super.addCustomEntries(store, passwdProps); + // TestManagedKeyStoreKeyProvider specific entries for (int i = 0; i < nCustodians; ++i) { String custodian = "custodian+ " + i; String alias = custodian + "-alias"; @@ -92,6 +121,17 @@ protected void addCustomEntries(KeyStore store, Properties passwdProps) throws E cust2alias, passwdProps); } + // Add custom namespace entries for testing + String customNamespace1 = "table1/cf1"; + String customNamespace2 = "table2"; + for (int i = 0; i < 2; ++i) { + String custodian = "ns-custodian+ " + i; + String alias = custodian + "-alias"; + String namespace = (i == 0) ? customNamespace1 : customNamespace2; + KeymetaTestUtils.addEntry(conf, 256, store, alias, custodian, withPasswordOnAlias, + namespaceCust2key, namespaceCust2alias, passwdProps, namespace); + } + clusterId = UUID.randomUUID().toString(); KeymetaTestUtils.addEntry(conf, 256, store, SYSTEM_KEY_ALIAS, clusterId, withPasswordOnAlias, cust2key, cust2alias, passwdProps); @@ -104,7 +144,7 @@ protected void addCustomEntries(KeyStore store, Properties passwdProps) throws E @Test public void testMissingConfig() throws Exception { - managedKeyProvider.initConfig(null); + managedKeyProvider.initConfig(null, null); RuntimeException ex = assertThrows(RuntimeException.class, () -> managedKeyProvider.getSystemKey(null)); assertEquals("initConfig is not called or config is null", ex.getMessage()); @@ -133,7 +173,8 @@ public void testGetGlobalCustodianKey() throws Exception { public void testGetInactiveKey() throws Exception { Bytes firstCust = cust2key.keySet().iterator().next(); String encCust = Base64.getEncoder().encodeToString(firstCust.get()); - conf.set(HConstants.CRYPTO_MANAGED_KEY_STORE_CONF_KEY_PREFIX + encCust + ".active", "false"); + conf.set(HConstants.CRYPTO_MANAGED_KEY_STORE_CONF_KEY_PREFIX + encCust + ".*.active", + "false"); ManagedKeyData keyData = managedKeyProvider.getManagedKey(firstCust.get(), ManagedKeyData.KEY_SPACE_GLOBAL); assertNotNull(keyData); @@ -154,12 +195,15 @@ public void testGetInvalidKey() throws Exception { public void testGetDisabledKey() throws Exception { byte[] invalidCust = new byte[] { 1, 2, 3 }; String invalidCustEnc = ManagedKeyProvider.encodeToStr(invalidCust); - conf.set(HConstants.CRYPTO_MANAGED_KEY_STORE_CONF_KEY_PREFIX + invalidCustEnc + ".active", + // For disabled keys, we need to configure both alias and active status + conf.set(HConstants.CRYPTO_MANAGED_KEY_STORE_CONF_KEY_PREFIX + invalidCustEnc + ".*.alias", + "disabled-alias"); + conf.set(HConstants.CRYPTO_MANAGED_KEY_STORE_CONF_KEY_PREFIX + invalidCustEnc + ".*.active", "false"); ManagedKeyData keyData = managedKeyProvider.getManagedKey(invalidCust, ManagedKeyData.KEY_SPACE_GLOBAL); assertNotNull(keyData); - assertKeyData(keyData, ManagedKeyState.DISABLED, null, invalidCust, null); + assertKeyData(keyData, ManagedKeyState.DISABLED, null, invalidCust, "disabled-alias"); } @Test @@ -193,19 +237,239 @@ public void testUnwrapDisabledKey() throws Exception { String invalidAlias = "invalidAlias"; byte[] invalidCust = new byte[] { 1, 2, 3 }; String invalidCustEnc = ManagedKeyProvider.encodeToStr(invalidCust); - conf.set(HConstants.CRYPTO_MANAGED_KEY_STORE_CONF_KEY_PREFIX + invalidCustEnc + ".active", + conf.set(HConstants.CRYPTO_MANAGED_KEY_STORE_CONF_KEY_PREFIX + invalidCustEnc + ".*.active", "false"); - String invalidMetadata = - ManagedKeyStoreKeyProvider.generateKeyMetadata(invalidAlias, invalidCustEnc); + String invalidMetadata = ManagedKeyStoreKeyProvider.generateKeyMetadata(invalidAlias, + invalidCustEnc, ManagedKeyData.KEY_SPACE_GLOBAL); ManagedKeyData keyData = managedKeyProvider.unwrapKey(invalidMetadata, null); assertNotNull(keyData); assertKeyData(keyData, ManagedKeyState.DISABLED, null, invalidCust, invalidAlias); } + @Test + public void testGetManagedKeyWithCustomNamespace() throws Exception { + String customNamespace1 = "table1/cf1"; + String customNamespace2 = "table2"; + int index = 0; + for (Bytes cust : namespaceCust2key.keySet()) { + String namespace = (index == 0) ? customNamespace1 : customNamespace2; + ManagedKeyData keyData = managedKeyProvider.getManagedKey(cust.get(), namespace); + assertKeyDataWithNamespace(keyData, ManagedKeyState.ACTIVE, + namespaceCust2key.get(cust).get(), cust.get(), namespaceCust2alias.get(cust), namespace); + index++; + } + } + + @Test + public void testGetManagedKeyWithCustomNamespaceInactive() throws Exception { + Bytes firstCust = namespaceCust2key.keySet().iterator().next(); + String customNamespace = "table1/cf1"; + String encCust = Base64.getEncoder().encodeToString(firstCust.get()); + // Set active status to false using the new namespace-aware format + conf.set(HConstants.CRYPTO_MANAGED_KEY_STORE_CONF_KEY_PREFIX + encCust + "." + customNamespace + + ".active", "false"); + + ManagedKeyData keyData = managedKeyProvider.getManagedKey(firstCust.get(), customNamespace); + assertNotNull(keyData); + assertKeyDataWithNamespace(keyData, ManagedKeyState.INACTIVE, + namespaceCust2key.get(firstCust).get(), firstCust.get(), namespaceCust2alias.get(firstCust), + customNamespace); + } + + @Test + public void testGetManagedKeyWithInvalidCustomNamespace() throws Exception { + byte[] invalidCustBytes = "invalid".getBytes(); + String customNamespace = "invalid/namespace"; + ManagedKeyData keyData = managedKeyProvider.getManagedKey(invalidCustBytes, customNamespace); + assertNotNull(keyData); + assertKeyDataWithNamespace(keyData, ManagedKeyState.FAILED, null, invalidCustBytes, null, + customNamespace); + } + + @Test + public void testNamespaceMismatchReturnsFailedKey() throws Exception { + // Use existing namespace key but request with different namespace + Bytes firstCust = namespaceCust2key.keySet().iterator().next(); + String requestedNamespace = "table2/cf2"; // Different namespace from what's configured! + + // Request key with different namespace - should fail + ManagedKeyData keyData = + managedKeyProvider.getManagedKey(firstCust.get(), requestedNamespace); + + assertNotNull(keyData); + assertEquals(ManagedKeyState.FAILED, keyData.getKeyState()); + assertNull(keyData.getTheKey()); + assertEquals(requestedNamespace, keyData.getKeyNamespace()); + assertEquals(firstCust, keyData.getKeyCustodian()); + } + + @Test + public void testNamespaceMatchReturnsKey() throws Exception { + // Use existing namespace key and request with matching namespace + Bytes firstCust = namespaceCust2key.keySet().iterator().next(); + String configuredNamespace = "table1/cf1"; // This matches our test setup + + ManagedKeyData keyData = + managedKeyProvider.getManagedKey(firstCust.get(), configuredNamespace); + + assertNotNull(keyData); + assertEquals(ManagedKeyState.ACTIVE, keyData.getKeyState()); + assertNotNull(keyData.getTheKey()); + assertEquals(configuredNamespace, keyData.getKeyNamespace()); + } + + @Test + public void testGlobalKeyAccessedWithWrongNamespaceFails() throws Exception { + // Get a global key (one from cust2key) + Bytes globalCust = cust2key.keySet().iterator().next(); + + // Try to access it with a custom namespace - should fail + String wrongNamespace = "table1/cf1"; + ManagedKeyData keyData = managedKeyProvider.getManagedKey(globalCust.get(), wrongNamespace); + + assertNotNull(keyData); + assertEquals(ManagedKeyState.FAILED, keyData.getKeyState()); + assertNull(keyData.getTheKey()); + assertEquals(wrongNamespace, keyData.getKeyNamespace()); + } + + @Test + public void testNamespaceKeyAccessedAsGlobalFails() throws Exception { + // Get a namespace-specific key + Bytes namespaceCust = namespaceCust2key.keySet().iterator().next(); + + // Try to access it as global - should fail + ManagedKeyData keyData = + managedKeyProvider.getManagedKey(namespaceCust.get(), ManagedKeyData.KEY_SPACE_GLOBAL); + + assertNotNull(keyData); + assertEquals(ManagedKeyState.FAILED, keyData.getKeyState()); + assertNull(keyData.getTheKey()); + assertEquals(ManagedKeyData.KEY_SPACE_GLOBAL, keyData.getKeyNamespace()); + } + + @Test + public void testMultipleNamespacesForSameCustodianFail() throws Exception { + // Use existing namespace custodian + Bytes namespaceCust = namespaceCust2key.keySet().iterator().next(); + String configuredNamespace = "table1/cf1"; // This matches our test setup + String differentNamespace = "table2"; + + // Verify we can access with configured namespace + ManagedKeyData keyData1 = + managedKeyProvider.getManagedKey(namespaceCust.get(), configuredNamespace); + assertEquals(ManagedKeyState.ACTIVE, keyData1.getKeyState()); + assertEquals(configuredNamespace, keyData1.getKeyNamespace()); + + // But accessing with different namespace should fail (even though it's the same custodian) + ManagedKeyData keyData2 = + managedKeyProvider.getManagedKey(namespaceCust.get(), differentNamespace); + assertEquals(ManagedKeyState.FAILED, keyData2.getKeyState()); + assertEquals(differentNamespace, keyData2.getKeyNamespace()); + } + + @Test + public void testNullNamespaceDefaultsToGlobal() throws Exception { + // Get a global key (one from cust2key) + Bytes globalCust = cust2key.keySet().iterator().next(); + + // Call getManagedKey with null namespace - should default to global and succeed + ManagedKeyData keyData = managedKeyProvider.getManagedKey(globalCust.get(), null); + + assertNotNull(keyData); + assertEquals(ManagedKeyState.ACTIVE, keyData.getKeyState()); + assertNotNull(keyData.getTheKey()); + assertEquals(ManagedKeyData.KEY_SPACE_GLOBAL, keyData.getKeyNamespace()); + } + + @Test + public void testFailedKeyContainsProperMetadataWithAlias() throws Exception { + // Use existing namespace key but request with different namespace + Bytes firstCust = namespaceCust2key.keySet().iterator().next(); + String wrongNamespace = "wrong/namespace"; + + // Request with wrong namespace - should fail but have proper metadata + ManagedKeyData keyData = managedKeyProvider.getManagedKey(firstCust.get(), wrongNamespace); + + assertNotNull(keyData); + assertEquals(ManagedKeyState.FAILED, keyData.getKeyState()); + assertNull(keyData.getTheKey()); + assertEquals(wrongNamespace, keyData.getKeyNamespace()); + + // Verify the failed key metadata - should have null alias since wrong namespace is requested + // This is the correct security behavior - don't leak alias information across namespaces + String expectedEncodedCust = Base64.getEncoder().encodeToString(firstCust.get()); + assertMetadataMatches(keyData.getKeyMetadata(), null, expectedEncodedCust, wrongNamespace); + } + + @Test + public void testBackwardsCompatibilityForGenerateKeyMetadata() { + String alias = "test-alias"; + String encodedCust = "dGVzdA=="; + + // Test the old method (should default to global namespace) + String oldMetadata = ManagedKeyStoreKeyProvider.generateKeyMetadata(alias, encodedCust); + + // Test the new method with explicit global namespace + String newMetadata = ManagedKeyStoreKeyProvider.generateKeyMetadata(alias, encodedCust, + ManagedKeyData.KEY_SPACE_GLOBAL); + + assertEquals( + "Old and new metadata generation should produce same result for global namespace", + oldMetadata, newMetadata); + + // Verify both contain the namespace field + Map oldMap = parseKeyMetadata(oldMetadata); + Map newMap = parseKeyMetadata(newMetadata); + + assertEquals(ManagedKeyData.KEY_SPACE_GLOBAL, + oldMap.get(ManagedKeyStoreKeyProvider.KEY_METADATA_NAMESPACE)); + assertEquals(ManagedKeyData.KEY_SPACE_GLOBAL, + newMap.get(ManagedKeyStoreKeyProvider.KEY_METADATA_NAMESPACE)); + } + private void assertKeyData(ManagedKeyData keyData, ManagedKeyState expKeyState, byte[] key, byte[] custBytes, String alias) throws Exception { + assertKeyDataWithNamespace(keyData, expKeyState, key, custBytes, alias, + ManagedKeyData.KEY_SPACE_GLOBAL); + } + + /** + * Helper method to parse key metadata JSON string into a Map + */ + @SuppressWarnings("unchecked") + private Map parseKeyMetadata(String keyMetadataStr) { + return GsonUtil.getDefaultInstance().fromJson(keyMetadataStr, HashMap.class); + } + + /** + * Helper method to assert metadata contents + */ + private void assertMetadataContains(Map metadata, String expectedAlias, + String expectedEncodedCust, String expectedNamespace) { + assertNotNull("Metadata should not be null", metadata); + assertEquals("Metadata should contain expected alias", expectedAlias, + metadata.get(KEY_METADATA_ALIAS)); + assertEquals("Metadata should contain expected encoded custodian", expectedEncodedCust, + metadata.get(KEY_METADATA_CUST)); + assertEquals("Metadata should contain expected namespace", expectedNamespace, + metadata.get(ManagedKeyStoreKeyProvider.KEY_METADATA_NAMESPACE)); + } + + /** + * Helper method to parse and assert metadata contents in one call + */ + private void assertMetadataMatches(String keyMetadataStr, String expectedAlias, + String expectedEncodedCust, String expectedNamespace) { + Map metadata = parseKeyMetadata(keyMetadataStr); + assertMetadataContains(metadata, expectedAlias, expectedEncodedCust, expectedNamespace); + } + + private void assertKeyDataWithNamespace(ManagedKeyData keyData, ManagedKeyState expKeyState, + byte[] key, byte[] custBytes, String alias, String expectedNamespace) throws Exception { assertNotNull(keyData); assertEquals(expKeyState, keyData.getKeyState()); + assertEquals(expectedNamespace, keyData.getKeyNamespace()); if (key == null) { assertNull(keyData.getTheKey()); } else { @@ -213,13 +477,12 @@ private void assertKeyData(ManagedKeyData keyData, ManagedKeyState expKeyState, assertEquals(key.length, keyBytes.length); assertEquals(new Bytes(key), keyBytes); } - Map keyMetadata = - GsonUtil.getDefaultInstance().fromJson(keyData.getKeyMetadata(), HashMap.class); - assertNotNull(keyMetadata); + + // Use helper method instead of duplicated parsing logic + String encodedCust = Base64.getEncoder().encodeToString(custBytes); + assertMetadataMatches(keyData.getKeyMetadata(), alias, encodedCust, expectedNamespace); + assertEquals(new Bytes(custBytes), keyData.getKeyCustodian()); - assertEquals(alias, keyMetadata.get(KEY_METADATA_ALIAS)); - assertEquals(Base64.getEncoder().encodeToString(custBytes), - keyMetadata.get(KEY_METADATA_CUST)); assertEquals(keyData, managedKeyProvider.unwrapKey(keyData.getKeyMetadata(), null)); } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/keymeta/KeyManagementBase.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/keymeta/KeyManagementBase.java index 957c3c8f726d..6fbd177437fe 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/keymeta/KeyManagementBase.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/keymeta/KeyManagementBase.java @@ -22,7 +22,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.io.crypto.Encryption; -import org.apache.hadoop.hbase.io.crypto.KeyProvider; import org.apache.hadoop.hbase.io.crypto.ManagedKeyData; import org.apache.hadoop.hbase.io.crypto.ManagedKeyProvider; import org.apache.hadoop.hbase.security.SecurityUtil; @@ -73,17 +72,11 @@ protected Configuration getConfiguration() { /** * A utility method for getting the managed key provider. - * @return the key provider - * @throws RuntimeException if no provider is configured or if the configured provider is not an - * instance of ManagedKeyProvider + * @return the managed key provider + * @throws RuntimeException if no provider is configured */ protected ManagedKeyProvider getKeyProvider() { - KeyProvider provider = Encryption.getKeyProvider(getConfiguration()); - if (!(provider instanceof ManagedKeyProvider)) { - throw new RuntimeException("KeyProvider: " + provider.getClass().getName() - + " expected to be of type ManagedKeyProvider"); - } - return (ManagedKeyProvider) provider; + return Encryption.getManagedKeyProvider(getConfiguration()); } /** diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java index 995f7fa6c47f..fde89d122e28 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java @@ -81,7 +81,6 @@ import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoderImpl; import org.apache.hadoop.hbase.io.hfile.HFileScanner; import org.apache.hadoop.hbase.io.hfile.InvalidHFileException; -import org.apache.hadoop.hbase.keymeta.KeyNamespaceUtil; import org.apache.hadoop.hbase.monitoring.MonitoredTask; import org.apache.hadoop.hbase.quotas.RegionSizeStore; import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext; @@ -337,9 +336,8 @@ protected HStore(final HRegion region, final ColumnFamilyDescriptor family, private StoreContext initializeStoreContext(ColumnFamilyDescriptor family) throws IOException { return new StoreContext.Builder().withBlockSize(family.getBlocksize()) - .withEncryptionContext(SecurityUtil.createEncryptionContext(conf, family, - region.getManagedKeyDataCache(), region.getSystemKeyCache(), - KeyNamespaceUtil.constructKeyNamespace(region.getTableDescriptor(), family))) + .withEncryptionContext(SecurityUtil.createEncryptionContext(conf, region.getTableDescriptor(), + family, region.getManagedKeyDataCache(), region.getSystemKeyCache())) .withBloomType(family.getBloomFilterType()).withCacheConfig(createCacheConf(family)) .withCellComparator(region.getTableDescriptor().isMetaTable() || conf .getBoolean(HRegion.USE_META_CELL_COMPARATOR, HRegion.DEFAULT_USE_META_CELL_COMPARATOR) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/SecurityUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/SecurityUtil.java index 2e6e4cb4f933..3fe2937e4d6a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/SecurityUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/SecurityUtil.java @@ -24,14 +24,18 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.io.crypto.Cipher; import org.apache.hadoop.hbase.io.crypto.Encryption; import org.apache.hadoop.hbase.io.crypto.ManagedKeyData; import org.apache.hadoop.hbase.io.hfile.FixedFileTrailer; +import org.apache.hadoop.hbase.keymeta.KeyNamespaceUtil; import org.apache.hadoop.hbase.keymeta.ManagedKeyDataCache; import org.apache.hadoop.hbase.keymeta.SystemKeyCache; import org.apache.yetus.audience.InterfaceAudience; import org.apache.yetus.audience.InterfaceStability; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Security related generic utility methods. @@ -39,6 +43,8 @@ @InterfaceAudience.Private @InterfaceStability.Evolving public class SecurityUtil { + private static final Logger LOG = LoggerFactory.getLogger(SecurityUtil.class); + /** * Get the user name from a principal */ @@ -61,74 +67,135 @@ public static String getPrincipalWithoutRealm(final String principal) { /** * Helper to create an encyption context with current encryption key, suitable for writes. * @param conf The current configuration. + * @param tableDescriptor The table descriptor. * @param family The current column descriptor. * @param managedKeyDataCache The managed key data cache. * @param systemKeyCache The system key cache. - * @param keyNamespace The key namespace. * @return The created encryption context. * @throws IOException if an encryption key for the column cannot be unwrapped * @throws IllegalStateException in case of encryption related configuration errors */ public static Encryption.Context createEncryptionContext(Configuration conf, - ColumnFamilyDescriptor family, ManagedKeyDataCache managedKeyDataCache, - SystemKeyCache systemKeyCache, String keyNamespace) throws IOException { + TableDescriptor tableDescriptor, ColumnFamilyDescriptor family, + ManagedKeyDataCache managedKeyDataCache, SystemKeyCache systemKeyCache) throws IOException { Encryption.Context cryptoContext = Encryption.Context.NONE; + boolean isKeyManagementEnabled = isKeyManagementEnabled(conf); String cipherName = family.getEncryptionType(); + String keyNamespace = null; // Will be set by fallback logic + LOG.debug("Creating encryption context for table: {} and column family: {}", + tableDescriptor.getTableName().getNameAsString(), family.getNameAsString()); if (cipherName != null) { if (!Encryption.isEncryptionEnabled(conf)) { throw new IllegalStateException("Encryption for family '" + family.getNameAsString() + "' configured with type '" + cipherName + "' but the encryption feature is disabled"); } + if (isKeyManagementEnabled && systemKeyCache == null) { + throw new IOException("Key management is enabled, but SystemKeyCache is null"); + } Cipher cipher = null; Key key = null; - ManagedKeyData kekKeyData = null; - if (isKeyManagementEnabled(conf)) { - kekKeyData = managedKeyDataCache.getActiveEntry(ManagedKeyData.KEY_GLOBAL_CUSTODIAN_BYTES, - keyNamespace); - // If no active key found in the specific namespace, try the global namespace - if (kekKeyData == null) { - kekKeyData = managedKeyDataCache.getActiveEntry(ManagedKeyData.KEY_GLOBAL_CUSTODIAN_BYTES, - ManagedKeyData.KEY_SPACE_GLOBAL); - keyNamespace = ManagedKeyData.KEY_SPACE_GLOBAL; - } - if (kekKeyData == null) { - throw new IOException( - "No active key found for custodian: " + ManagedKeyData.KEY_GLOBAL_CUSTODIAN - + " in namespaces: " + keyNamespace + " and " + ManagedKeyData.KEY_SPACE_GLOBAL); - } - if ( - conf.getBoolean(HConstants.CRYPTO_MANAGED_KEYS_LOCAL_KEY_GEN_PER_FILE_ENABLED_CONF_KEY, - HConstants.CRYPTO_MANAGED_KEYS_LOCAL_KEY_GEN_PER_FILE_DEFAULT_ENABLED) - ) { - cipher = - getCipherIfValid(conf, cipherName, kekKeyData.getTheKey(), family.getNameAsString()); - } else { - key = kekKeyData.getTheKey(); - kekKeyData = systemKeyCache.getLatestSystemKey(); + ManagedKeyData kekKeyData = + isKeyManagementEnabled ? systemKeyCache.getLatestSystemKey() : null; + + // Scenario 1: If family has a key, unwrap it and use that as DEK. + byte[] familyKeyBytes = family.getEncryptionKey(); + if (familyKeyBytes != null) { + try { + if (isKeyManagementEnabled) { + // Scenario 1a: If key management is enabled, use STK for both unwrapping and KEK. + key = EncryptionUtil.unwrapKey(conf, null, familyKeyBytes, kekKeyData.getTheKey()); + } else { + // Scenario 1b: If key management is disabled, unwrap the key using master key. + key = EncryptionUtil.unwrapKey(conf, familyKeyBytes); + } + if (LOG.isDebugEnabled()) { + LOG.debug("Scenario 1: Use family key for namespace {} cipher: {} " + + "key management enabled: {}", keyNamespace, cipherName, isKeyManagementEnabled); + } + } catch (KeyException e) { + throw new IOException(e); } } else { - byte[] keyBytes = family.getEncryptionKey(); - if (keyBytes != null) { - // Family provides specific key material - key = EncryptionUtil.unwrapKey(conf, keyBytes); + if (isKeyManagementEnabled) { + boolean localKeyGenEnabled = + conf.getBoolean(HConstants.CRYPTO_MANAGED_KEYS_LOCAL_KEY_GEN_PER_FILE_ENABLED_CONF_KEY, + HConstants.CRYPTO_MANAGED_KEYS_LOCAL_KEY_GEN_PER_FILE_DEFAULT_ENABLED); + // Implement 4-step fallback logic for key namespace resolution in the order of + // 1. CF KEY_NAMESPACE attribute + // 2. Constructed namespace + // 3. Table name + // 4. Global namespace + String[] candidateNamespaces = { family.getEncryptionKeyNamespace(), + KeyNamespaceUtil.constructKeyNamespace(tableDescriptor, family), + tableDescriptor.getTableName().getNameAsString(), ManagedKeyData.KEY_SPACE_GLOBAL }; + + ManagedKeyData activeKeyData = null; + for (String candidate : candidateNamespaces) { + if (candidate != null) { + // Log information on the table and column family we are looking for the active key in + LOG.debug("Looking for active key for table: {} and column family: {}", + tableDescriptor.getTableName().getNameAsString(), family.getNameAsString()); + activeKeyData = managedKeyDataCache + .getActiveEntry(ManagedKeyData.KEY_GLOBAL_CUSTODIAN_BYTES, candidate); + if (activeKeyData != null) { + keyNamespace = candidate; + break; + } + } + } + + // Scenario 2: There is an active key + if (activeKeyData != null) { + if (!localKeyGenEnabled) { + // Scenario 2a: Use active key as DEK and latest STK as KEK + key = activeKeyData.getTheKey(); + } else { + // Scenario 2b: Use active key as KEK and generate local key as DEK + kekKeyData = activeKeyData; + // TODO: Use the active key as a seed to generate the local key instead of + // random generation + cipher = getCipherIfValid(conf, cipherName, activeKeyData.getTheKey(), + family.getNameAsString()); + } + if (LOG.isDebugEnabled()) { + LOG.debug( + "Scenario 2: Use active key for namespace {} cipher: {} " + + "localKeyGenEnabled: {} for table: {} and column family: {}", + keyNamespace, cipherName, localKeyGenEnabled, + tableDescriptor.getTableName().getNameAsString(), family.getNameAsString(), + activeKeyData.getKeyNamespace()); + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Scenario 3a: No active key found for table: {} and column family: {}", + tableDescriptor.getTableName().getNameAsString(), family.getNameAsString()); + } + // Scenario 3a: Do nothing, let a random key be generated as DEK and if key management + // is enabled, let STK be used as KEK. + } } else { - cipher = getCipherIfValid(conf, cipherName, null, null); + // Scenario 3b: Do nothing, let a random key be generated as DEK, let STK be used as KEK. + if (LOG.isDebugEnabled()) { + LOG.debug( + "Scenario 3b: Key management is disabled and no ENCRYPTION_KEY attribute " + + "set for table: {} and column family: {}", + tableDescriptor.getTableName().getNameAsString(), family.getNameAsString()); + } } } - if (key != null || cipher != null) { - if (key == null) { - // Family does not provide key material, create a random key - key = cipher.getRandomKey(); - } - if (cipher == null) { - cipher = getCipherIfValid(conf, cipherName, key, family.getNameAsString()); - } - cryptoContext = Encryption.newContext(conf); - cryptoContext.setCipher(cipher); - cryptoContext.setKey(key); - cryptoContext.setKeyNamespace(keyNamespace); - cryptoContext.setKEKData(kekKeyData); + + if (cipher == null) { + cipher = + getCipherIfValid(conf, cipherName, key, key == null ? null : family.getNameAsString()); + } + if (key == null) { + key = cipher.getRandomKey(); } + cryptoContext = Encryption.newContext(conf); + cryptoContext.setCipher(cipher); + cryptoContext.setKey(key); + cryptoContext.setKeyNamespace(keyNamespace); + cryptoContext.setKEKData(kekKeyData); } return cryptoContext; } @@ -149,15 +216,38 @@ public static Encryption.Context createEncryptionContext(Configuration conf, Pat ManagedKeyData kekKeyData = null; byte[] keyBytes = trailer.getEncryptionKey(); Encryption.Context cryptoContext = Encryption.Context.NONE; + if (LOG.isDebugEnabled()) { + LOG.debug("Creating encryption context for path: {}", path); + } // Check for any key material available if (keyBytes != null) { cryptoContext = Encryption.newContext(conf); Key kek = null; - // When the KEK medata is available, we will try to unwrap the encrypted key using the KEK, - // otherwise we will use the system keys starting from the latest to the oldest. - if (trailer.getKEKMetadata() != null) { + + // When there is key material, determine the appropriate KEK + boolean isKeyManagementEnabled = isKeyManagementEnabled(conf); + if (((trailer.getKEKChecksum() != 0L) || isKeyManagementEnabled) && systemKeyCache == null) { + throw new IOException("SystemKeyCache can't be null when using key management feature"); + } + if ((trailer.getKEKChecksum() != 0L && !isKeyManagementEnabled)) { + throw new IOException( + "Seeing newer trailer with KEK checksum, but key management is disabled"); + } + + // Try STK lookup first if checksum is available and system key cache is not null. + if (trailer.getKEKChecksum() != 0L && trailer.getKeyNamespace() == null) { + ManagedKeyData systemKeyData = + systemKeyCache.getSystemKeyByChecksum(trailer.getKEKChecksum()); + if (systemKeyData != null) { + kek = systemKeyData.getTheKey(); + kekKeyData = systemKeyData; + } + } + + // If STK lookup failed or no checksum available, try managed key lookup using metadata + if (kek == null && trailer.getKEKMetadata() != null) { if (managedKeyDataCache == null) { - throw new IOException("Key management is enabled, but ManagedKeyDataCache is null"); + throw new IOException("KEK metadata is available, but ManagedKeyDataCache is null"); } Throwable cause = null; try { @@ -172,21 +262,17 @@ public static Encryption.Context createEncryptionContext(Configuration conf, Pat "Failed to get key data for KEK metadata: " + trailer.getKEKMetadata(), cause); } kek = kekKeyData.getTheKey(); - } else { - if (SecurityUtil.isKeyManagementEnabled(conf)) { - if (systemKeyCache == null) { - throw new IOException("Key management is enabled, but SystemKeyCache is null"); - } - ManagedKeyData systemKeyData = - systemKeyCache.getSystemKeyByChecksum(trailer.getKEKChecksum()); - if (systemKeyData == null) { - throw new IOException( - "Failed to get system key by checksum: " + trailer.getKEKChecksum()); - } - kek = systemKeyData.getTheKey(); - kekKeyData = systemKeyData; + } else if (kek == null && isKeyManagementEnabled) { + // No checksum or metadata available, fall back to latest system key for backwards + // compatibility + ManagedKeyData systemKeyData = systemKeyCache.getLatestSystemKey(); + if (systemKeyData == null) { + throw new IOException("Failed to get latest system key"); } + kek = systemKeyData.getTheKey(); + kekKeyData = systemKeyData; } + Key key; if (kek != null) { try { @@ -202,6 +288,7 @@ public static Encryption.Context createEncryptionContext(Configuration conf, Pat Cipher cipher = getCipherIfValid(conf, key.getAlgorithm(), key, null); cryptoContext.setCipher(cipher); cryptoContext.setKey(key); + cryptoContext.setKeyNamespace(trailer.getKeyNamespace()); cryptoContext.setKEKData(kekKeyData); } return cryptoContext; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/EncryptionTest.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/EncryptionTest.java index 192343ae41d3..eb4d72c7745f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/EncryptionTest.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/EncryptionTest.java @@ -28,7 +28,9 @@ import org.apache.hadoop.hbase.io.crypto.DefaultCipherProvider; import org.apache.hadoop.hbase.io.crypto.Encryption; import org.apache.hadoop.hbase.io.crypto.KeyStoreKeyProvider; +import org.apache.hadoop.hbase.io.crypto.ManagedKeyStoreKeyProvider; import org.apache.hadoop.hbase.security.EncryptionUtil; +import org.apache.hadoop.hbase.security.SecurityUtil; import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -48,12 +50,23 @@ private EncryptionTest() { * Check that the configured key provider can be loaded and initialized, or throw an exception. */ public static void testKeyProvider(final Configuration conf) throws IOException { - String providerClassName = - conf.get(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, KeyStoreKeyProvider.class.getName()); + boolean isKeyManagementEnabled = SecurityUtil.isKeyManagementEnabled(conf); + String providerClassName; + if (isKeyManagementEnabled) { + providerClassName = conf.get(HConstants.CRYPTO_MANAGED_KEYPROVIDER_CONF_KEY, + ManagedKeyStoreKeyProvider.class.getName()); + } else { + providerClassName = + conf.get(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, KeyStoreKeyProvider.class.getName()); + } Boolean result = keyProviderResults.get(providerClassName); if (result == null) { try { - Encryption.getKeyProvider(conf); + if (isKeyManagementEnabled) { + Encryption.getManagedKeyProvider(conf); + } else { + Encryption.getKeyProvider(conf); + } keyProviderResults.put(providerClassName, true); } catch (Exception e) { // most likely a RuntimeException keyProviderResults.put(providerClassName, false); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/keymeta/ManagedKeyProviderInterceptor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/keymeta/ManagedKeyProviderInterceptor.java index 3053e72ecea7..c91539b7ed68 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/keymeta/ManagedKeyProviderInterceptor.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/keymeta/ManagedKeyProviderInterceptor.java @@ -35,8 +35,8 @@ public ManagedKeyProviderInterceptor() { } @Override - public void initConfig(Configuration conf) { - spy.initConfig(conf); + public void initConfig(Configuration conf, String providerParameters) { + spy.initConfig(conf, providerParameters); } @Override diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/keymeta/ManagedKeyTestBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/keymeta/ManagedKeyTestBase.java index f3b2e2ca1ade..3c337ce72131 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/keymeta/ManagedKeyTestBase.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/keymeta/ManagedKeyTestBase.java @@ -19,34 +19,108 @@ import org.apache.hadoop.hbase.HBaseTestingUtil; import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.io.crypto.Encryption; import org.apache.hadoop.hbase.io.crypto.ManagedKeyProvider; import org.apache.hadoop.hbase.io.crypto.MockManagedKeyProvider; import org.junit.After; import org.junit.Before; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class ManagedKeyTestBase { + private static final Logger LOG = LoggerFactory.getLogger(ManagedKeyTestBase.class); + protected HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil(); @Before public void setUp() throws Exception { - TEST_UTIL.getConfiguration().set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, - getKeyProviderClass().getName()); - TEST_UTIL.getConfiguration().set(HConstants.CRYPTO_MANAGED_KEYS_ENABLED_CONF_KEY, "true"); - TEST_UTIL.getConfiguration().set("hbase.coprocessor.master.classes", - KeymetaServiceEndpoint.class.getName()); + if (isWithKeyManagement()) { + TEST_UTIL.getConfiguration().set(HConstants.CRYPTO_MANAGED_KEYPROVIDER_CONF_KEY, + getKeyProviderClass().getName()); + TEST_UTIL.getConfiguration().set(HConstants.CRYPTO_MANAGED_KEYS_ENABLED_CONF_KEY, "true"); + TEST_UTIL.getConfiguration().set("hbase.coprocessor.master.classes", + KeymetaServiceEndpoint.class.getName()); + } + + // Start the minicluster if needed + if (isWithMiniClusterStart()) { + LOG.info("\n\nManagedKeyTestBase.setUp: Starting minicluster\n"); + startMiniCluster(); + LOG.info("\n\nManagedKeyTestBase.setUp: Minicluster successfully started\n"); + } + } - // Start the minicluster + protected void startMiniCluster() throws Exception { + startMiniCluster(getSystemTableNameToWaitFor()); + } + + protected void startMiniCluster(TableName tableNameToWaitFor) throws Exception { TEST_UTIL.startMiniCluster(1); + waitForMasterInitialization(tableNameToWaitFor); + } + + protected void restartMiniCluster() throws Exception { + restartMiniCluster(getSystemTableNameToWaitFor()); + } + + protected void restartMiniCluster(TableName tableNameToWaitFor) throws Exception { + LOG.info("\n\nManagedKeyTestBase.restartMiniCluster: Flushing caches\n"); + TEST_UTIL.flush(); + + LOG.info("\n\nManagedKeyTestBase.restartMiniCluster: Shutting down cluster\n"); + TEST_UTIL.shutdownMiniHBaseCluster(); + + LOG.info("\n\nManagedKeyTestBase.restartMiniCluster: Sleeping a bit\n"); + Thread.sleep(2000); + + LOG.info("\n\nManagedKeyTestBase.restartMiniCluster: Starting the cluster back up\n"); + TEST_UTIL.restartHBaseCluster(1); + + waitForMasterInitialization(tableNameToWaitFor); + } + + private void waitForMasterInitialization(TableName tableNameToWaitFor) throws Exception { + LOG.info( + "\n\nManagedKeyTestBase.waitForMasterInitialization: Waiting for master initialization\n"); TEST_UTIL.waitFor(60000, () -> TEST_UTIL.getMiniHBaseCluster().getMaster().isInitialized()); - TEST_UTIL.waitUntilAllRegionsAssigned(KeymetaTableAccessor.KEY_META_TABLE_NAME); + + LOG.info( + "\n\nManagedKeyTestBase.waitForMasterInitialization: Waiting for regions to be assigned\n"); + TEST_UTIL.waitUntilAllRegionsAssigned(tableNameToWaitFor); + LOG.info("\n\nManagedKeyTestBase.waitForMasterInitialization: Regions assigned\n"); } @After public void tearDown() throws Exception { + LOG.info("\n\nManagedKeyTestBase.tearDown: Shutting down cluster\n"); TEST_UTIL.shutdownMiniCluster(); + LOG.info("\n\nManagedKeyTestBase.tearDown: Cluster successfully shut down\n"); + // Clear the provider cache to avoid test interference + Encryption.clearKeyProviderCache(); } protected Class getKeyProviderClass() { return MockManagedKeyProvider.class; } + + protected boolean isWithKeyManagement() { + return true; + } + + protected boolean isWithMiniClusterStart() { + return true; + } + + protected TableName getSystemTableNameToWaitFor() { + return KeymetaTableAccessor.KEY_META_TABLE_NAME; + } + + /** + * Useful hook to enable setting a breakpoint while debugging ruby tests, just log a message and + * you can even have a conditional breakpoint. + */ + protected void logMessage(String msg) { + LOG.info(msg); + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/keymeta/TestKeyManagementBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/keymeta/TestKeyManagementBase.java index 8ae91de6588f..3f6ddad6a1ee 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/keymeta/TestKeyManagementBase.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/keymeta/TestKeyManagementBase.java @@ -43,7 +43,7 @@ public class TestKeyManagementBase { public void testGetKeyProviderWithInvalidProvider() throws Exception { // Setup configuration with a non-ManagedKeyProvider Configuration conf = new Configuration(); - conf.set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, + conf.set(HConstants.CRYPTO_MANAGED_KEYPROVIDER_CONF_KEY, "org.apache.hadoop.hbase.keymeta.DummyKeyProvider"); MasterServices mockServer = mock(MasterServices.class); @@ -52,16 +52,21 @@ public void testGetKeyProviderWithInvalidProvider() throws Exception { final KeyManagementBase keyMgmt = new TestKeyManagement(mockServer); assertEquals(mockServer, keyMgmt.getKeyManagementService()); - // Should throw RuntimeException when provider is not ManagedKeyProvider + // Should throw RuntimeException when provider cannot be cast to ManagedKeyProvider RuntimeException exception = assertThrows(RuntimeException.class, () -> { keyMgmt.getKeyProvider(); }); - assertTrue(exception.getMessage().contains("expected to be of type ManagedKeyProvider")); + // The error message will be about ClassCastException since DummyKeyProvider doesn't implement + // ManagedKeyProvider + assertTrue(exception.getMessage().contains("ClassCastException") + || exception.getCause() instanceof ClassCastException); + exception = assertThrows(RuntimeException.class, () -> { KeyManagementBase keyMgmt2 = new TestKeyManagement(conf); keyMgmt2.getKeyProvider(); }); - assertTrue(exception.getMessage().contains("expected to be of type ManagedKeyProvider")); + assertTrue(exception.getMessage().contains("ClassCastException") + || exception.getCause() instanceof ClassCastException); assertThrows(IllegalArgumentException.class, () -> { Configuration configuration = null; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/keymeta/TestKeyManagementService.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/keymeta/TestKeyManagementService.java index 3fe669f90d80..bfd8be319895 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/keymeta/TestKeyManagementService.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/keymeta/TestKeyManagementService.java @@ -63,8 +63,11 @@ public class TestKeyManagementService { @Before public void setUp() throws Exception { + // Clear provider cache to avoid interference from other tests + Encryption.clearKeyProviderCache(); conf.set(HConstants.CRYPTO_MANAGED_KEYS_ENABLED_CONF_KEY, "true"); - conf.set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, MockManagedKeyProvider.class.getName()); + conf.set(HConstants.CRYPTO_MANAGED_KEYPROVIDER_CONF_KEY, + MockManagedKeyProvider.class.getName()); conf.set(HConstants.HBASE_ORIGINAL_DIR, "/tmp/hbase"); } @@ -72,7 +75,8 @@ public void setUp() throws Exception { public void testDefaultKeyManagementServiceCreation() throws IOException { // SystemKeyCache needs at least one valid key to be created, so setting up a mock FS that // returns a mock file that returns a known mocked key metadata. - MockManagedKeyProvider provider = (MockManagedKeyProvider) Encryption.getKeyProvider(conf); + MockManagedKeyProvider provider = + (MockManagedKeyProvider) Encryption.getManagedKeyProvider(conf); ManagedKeyData keyData = provider.getManagedKey("system".getBytes(), ManagedKeyData.KEY_SPACE_GLOBAL); String fileName = SYSTEM_KEY_FILE_PREFIX + "1"; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/keymeta/TestKeymetaTableAccessor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/keymeta/TestKeymetaTableAccessor.java index b695dedcdf98..2afa235007c7 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/keymeta/TestKeymetaTableAccessor.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/keymeta/TestKeymetaTableAccessor.java @@ -121,7 +121,8 @@ public void setUp() throws Exception { closeableMocks = MockitoAnnotations.openMocks(this); conf.set(HConstants.CRYPTO_MANAGED_KEYS_ENABLED_CONF_KEY, "true"); - conf.set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, MockManagedKeyProvider.class.getName()); + conf.set(HConstants.CRYPTO_MANAGED_KEYPROVIDER_CONF_KEY, + MockManagedKeyProvider.class.getName()); when(server.getConnection()).thenReturn(connection); when(connection.getTable(KeymetaTableAccessor.KEY_META_TABLE_NAME)).thenReturn(table); @@ -131,7 +132,7 @@ public void setUp() throws Exception { accessor = new KeymetaTableAccessor(server); managedKeyProvider = new MockManagedKeyProvider(); - managedKeyProvider.initConfig(conf); + managedKeyProvider.initConfig(conf, ""); latestSystemKey = managedKeyProvider.getSystemKey("system-id".getBytes()); when(systemKeyCache.getLatestSystemKey()).thenReturn(latestSystemKey); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/keymeta/TestManagedKeyDataCache.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/keymeta/TestManagedKeyDataCache.java index 807586a9a476..6e2eef1f67a7 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/keymeta/TestManagedKeyDataCache.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/keymeta/TestManagedKeyDataCache.java @@ -132,7 +132,7 @@ public void setUp() { Encryption.clearKeyProviderCache(); conf.set(HConstants.CRYPTO_MANAGED_KEYS_ENABLED_CONF_KEY, "true"); - conf.set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, providerClass.getName()); + conf.set(HConstants.CRYPTO_MANAGED_KEYPROVIDER_CONF_KEY, providerClass.getName()); // Configure the server mock to return the configuration when(server.getConfiguration()).thenReturn(conf); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/keymeta/TestManagedKeymeta.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/keymeta/TestManagedKeymeta.java index 52659b6cf2a4..63f05e7ee5e9 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/keymeta/TestManagedKeymeta.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/keymeta/TestManagedKeymeta.java @@ -68,7 +68,7 @@ public void testEnableOverRPC() throws Exception { private void doTestEnable(KeymetaAdmin adminClient) throws IOException, KeyException { HMaster master = TEST_UTIL.getHBaseCluster().getMaster(); MockManagedKeyProvider managedKeyProvider = - (MockManagedKeyProvider) Encryption.getKeyProvider(master.getConfiguration()); + (MockManagedKeyProvider) Encryption.getManagedKeyProvider(master.getConfiguration()); String cust = "cust1"; String encodedCust = ManagedKeyProvider.encodeToStr(cust.getBytes()); List managedKeyStates = diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestKeymetaAdminImpl.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestKeymetaAdminImpl.java index 6592238add50..a2cb14223e17 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestKeymetaAdminImpl.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestKeymetaAdminImpl.java @@ -53,6 +53,7 @@ import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; +import org.junit.After; import org.junit.Before; import org.junit.ClassRule; import org.junit.Rule; @@ -97,7 +98,8 @@ public void setUp() throws Exception { fs = testRootDir.getFileSystem(conf); conf.set(HConstants.CRYPTO_MANAGED_KEYS_ENABLED_CONF_KEY, "true"); - conf.set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, MockManagedKeyProvider.class.getName()); + conf.set(HConstants.CRYPTO_MANAGED_KEYPROVIDER_CONF_KEY, + MockManagedKeyProvider.class.getName()); when(mockServer.getKeyManagementService()).thenReturn(mockServer); when(mockServer.getFileSystem()).thenReturn(mockFileSystem); @@ -105,6 +107,12 @@ public void setUp() throws Exception { keymetaAdmin = new KeymetaAdminImplForTest(mockServer, keymetaAccessor); } + @After + public void tearDown() throws Exception { + // Clear the provider cache to avoid test interference + Encryption.clearKeyProviderCache(); + } + @RunWith(BlockJUnit4ClassRunner.class) @Category({ MasterTests.class, SmallTests.class }) public static class TestWhenDisabled extends TestKeymetaAdminImpl { @@ -141,7 +149,7 @@ public static class TestAdminImpl extends TestKeymetaAdminImpl { @Parameter(2) public boolean isNullKey; - @Parameters(name = "{index},keySpace={1},keyState={2}") + @Parameters(name = "{index},keySpace={0},keyState={1}") public static Collection data() { return Arrays.asList(new Object[][] { { KEY_SPACE_GLOBAL, ACTIVE, false }, { "ns1", ACTIVE, false }, { KEY_SPACE_GLOBAL, FAILED, true }, @@ -151,7 +159,7 @@ public static Collection data() { @Test public void testEnableAndGet() throws Exception { MockManagedKeyProvider managedKeyProvider = - (MockManagedKeyProvider) Encryption.getKeyProvider(conf); + (MockManagedKeyProvider) Encryption.getManagedKeyProvider(conf); managedKeyProvider.setMockedKeyState(CUST, keyState); when(keymetaAccessor.getActiveKey(CUST.getBytes(), keySpace)) .thenReturn(managedKeyProvider.getManagedKey(CUST.getBytes(), keySpace)); @@ -211,7 +219,7 @@ public static Collection data() { @Test public void test() throws Exception { MockManagedKeyProvider managedKeyProvider = - (MockManagedKeyProvider) Encryption.getKeyProvider(conf); + (MockManagedKeyProvider) Encryption.getManagedKeyProvider(conf); String cust = "invalidcust1"; String encodedCust = ManagedKeyProvider.encodeToStr(cust.getBytes()); managedKeyProvider.setMockedKey(cust, null, keySpace); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestSystemKeyManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestSystemKeyManager.java index e73c181a74fd..54bfb5e0a120 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestSystemKeyManager.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestSystemKeyManager.java @@ -26,7 +26,6 @@ import java.security.Key; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.io.crypto.Encryption; -import org.apache.hadoop.hbase.io.crypto.KeyProvider; import org.apache.hadoop.hbase.io.crypto.ManagedKeyData; import org.apache.hadoop.hbase.io.crypto.ManagedKeyProvider; import org.apache.hadoop.hbase.io.crypto.ManagedKeyState; @@ -51,7 +50,7 @@ public class TestSystemKeyManager extends ManagedKeyTestBase { @Test public void testSystemKeyInitializationAndRotation() throws Exception { HMaster master = TEST_UTIL.getHBaseCluster().getMaster(); - KeyProvider keyProvider = Encryption.getKeyProvider(master.getConfiguration()); + ManagedKeyProvider keyProvider = Encryption.getManagedKeyProvider(master.getConfiguration()); assertNotNull(keyProvider); assertTrue(keyProvider instanceof ManagedKeyProvider); assertTrue(keyProvider instanceof MockManagedKeyProvider); @@ -85,7 +84,7 @@ public void testSystemKeyInitializationAndRotation() throws Exception { @Test public void testWithInvalidSystemKey() throws Exception { HMaster master = TEST_UTIL.getHBaseCluster().getMaster(); - KeyProvider keyProvider = Encryption.getKeyProvider(master.getConfiguration()); + ManagedKeyProvider keyProvider = Encryption.getManagedKeyProvider(master.getConfiguration()); MockManagedKeyProvider pbeKeyProvider = (MockManagedKeyProvider) keyProvider; // Test startup failure when the cluster key is INACTIVE diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/TestSecurityUtil.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/TestSecurityUtil.java index ca2f8088a786..2077673fde8a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/TestSecurityUtil.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/TestSecurityUtil.java @@ -39,7 +39,9 @@ import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseTestingUtil; import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.io.crypto.Cipher; import org.apache.hadoop.hbase.io.crypto.CipherProvider; import org.apache.hadoop.hbase.io.crypto.Encryption; @@ -47,6 +49,7 @@ import org.apache.hadoop.hbase.io.crypto.ManagedKeyData; import org.apache.hadoop.hbase.io.crypto.MockAesKeyProvider; import org.apache.hadoop.hbase.io.hfile.FixedFileTrailer; +import org.apache.hadoop.hbase.keymeta.KeyNamespaceUtil; import org.apache.hadoop.hbase.keymeta.ManagedKeyDataCache; import org.apache.hadoop.hbase.keymeta.SystemKeyCache; import org.apache.hadoop.hbase.testclassification.SecurityTests; @@ -92,6 +95,7 @@ public class TestSecurityUtil { protected HBaseTestingUtil testUtil; protected Path testPath; protected ColumnFamilyDescriptor mockFamily; + protected TableDescriptor mockTableDescriptor; protected ManagedKeyDataCache mockManagedKeyDataCache; protected SystemKeyCache mockSystemKeyCache; protected FixedFileTrailer mockTrailer; @@ -99,6 +103,7 @@ public class TestSecurityUtil { protected Key testKey; protected byte[] testWrappedKey; protected Key kekKey; + protected String testTableNamespace; /** * Configuration builder for setting up different encryption test scenarios. @@ -116,8 +121,8 @@ public TestConfigBuilder withEncryptionEnabled(boolean enabled) { return this; } - public TestConfigBuilder withKeyManagement(boolean enabled, boolean localKeyGen) { - this.keyManagementEnabled = enabled; + public TestConfigBuilder withKeyManagement(boolean localKeyGen) { + this.keyManagementEnabled = true; this.localKeyGenEnabled = localKeyGen; return this; } @@ -152,7 +157,7 @@ protected void setUpEncryptionConfig() { // Set up real encryption configuration using default AES cipher conf.setBoolean(Encryption.CRYPTO_ENABLED_CONF_KEY, true); conf.set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, MockAesKeyProvider.class.getName()); - conf.set(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, "hbase"); + conf.set(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, HBASE_KEY); // Enable key caching conf.set(HConstants.CRYPTO_KEYPROVIDER_PARAMETERS_KEY, "true"); // Use DefaultCipherProvider for real AES encryption functionality @@ -215,8 +220,8 @@ protected void assertExceptionContains(Class expectedTy protected void assertEncryptionContextThrowsForWrites(Class expectedType, String expectedMessage) { Exception exception = assertThrows(Exception.class, () -> { - SecurityUtil.createEncryptionContext(conf, mockFamily, mockManagedKeyDataCache, - mockSystemKeyCache, TEST_NAMESPACE); + SecurityUtil.createEncryptionContext(conf, mockTableDescriptor, mockFamily, + mockManagedKeyDataCache, mockSystemKeyCache); }); assertTrue("Expected exception type: " + expectedType.getName() + ", but got: " + exception.getClass().getName(), expectedType.isInstance(exception)); @@ -244,6 +249,7 @@ public void setUp() throws Exception { // Setup mocks (only for objects that don't have encryption logic) mockFamily = mock(ColumnFamilyDescriptor.class); + mockTableDescriptor = mock(TableDescriptor.class); mockManagedKeyDataCache = mock(ManagedKeyDataCache.class); mockSystemKeyCache = mock(SystemKeyCache.class); mockTrailer = mock(FixedFileTrailer.class); @@ -255,8 +261,13 @@ public void setUp() throws Exception { // Configure mocks when(mockFamily.getEncryptionType()).thenReturn(AES_CIPHER); when(mockFamily.getNameAsString()).thenReturn(TEST_FAMILY); + when(mockFamily.getEncryptionKeyNamespace()).thenReturn(null); // Default to null for fallback + // logic + when(mockTableDescriptor.getTableName()).thenReturn(TableName.valueOf("test:table")); when(mockManagedKeyData.getTheKey()).thenReturn(testKey); + testTableNamespace = KeyNamespaceUtil.constructKeyNamespace(mockTableDescriptor, mockFamily); + // Set up default encryption config setUpEncryptionConfig(); @@ -267,6 +278,12 @@ public void setUp() throws Exception { testWrappedKey = EncryptionUtil.wrapKey(conf, null, key, kekKey); } + private static byte[] createRandomWrappedKey(Configuration conf) throws IOException { + Cipher cipher = Encryption.getCipher(conf, "AES"); + Key key = cipher.getRandomKey(); + return EncryptionUtil.wrapKey(conf, HBASE_KEY, key); + } + @RunWith(BlockJUnit4ClassRunner.class) @Category({ SecurityTests.class, SmallTests.class }) public static class TestBasic extends TestSecurityUtil { @@ -327,8 +344,8 @@ public static class TestCreateEncryptionContext_ForWrites extends TestSecurityUt public void testWithNoEncryptionOnFamily() throws IOException { when(mockFamily.getEncryptionType()).thenReturn(null); - Encryption.Context result = SecurityUtil.createEncryptionContext(conf, mockFamily, - mockManagedKeyDataCache, mockSystemKeyCache, "test-namespace"); + Encryption.Context result = SecurityUtil.createEncryptionContext(conf, mockTableDescriptor, + mockFamily, mockManagedKeyDataCache, mockSystemKeyCache); assertEquals(Encryption.Context.NONE, result); } @@ -342,20 +359,47 @@ public void testWithEncryptionDisabled() throws IOException { @Test public void testWithKeyManagement_LocalKeyGen() throws IOException { - configBuilder().withKeyManagement(true, true).apply(conf); - setupManagedKeyDataCache(TEST_NAMESPACE, mockManagedKeyData); + configBuilder().withKeyManagement(true).apply(conf); + setupManagedKeyDataCache(testTableNamespace, mockManagedKeyData); - Encryption.Context result = SecurityUtil.createEncryptionContext(conf, mockFamily, - mockManagedKeyDataCache, mockSystemKeyCache, TEST_NAMESPACE); + Encryption.Context result = SecurityUtil.createEncryptionContext(conf, mockTableDescriptor, + mockFamily, mockManagedKeyDataCache, mockSystemKeyCache); verifyContext(result); } @Test - public void testWithKeyManagement_NoActiveKey() throws IOException { - configBuilder().withKeyManagement(true, false).apply(conf); - setupManagedKeyDataCache(TEST_NAMESPACE, ManagedKeyData.KEY_SPACE_GLOBAL, null); - assertEncryptionContextThrowsForWrites(IOException.class, "No active key found"); + public void testWithKeyManagement_NoActiveKey_NoSystemKeyCache() throws IOException { + // Test backwards compatibility: when no active key found and system cache is null, should + // throw + configBuilder().withKeyManagement(false).apply(conf); + setupManagedKeyDataCache(testTableNamespace, ManagedKeyData.KEY_SPACE_GLOBAL, null); + when(mockFamily.getEncryptionKey()).thenReturn(null); + + // With null system key cache, should still throw IOException + Exception exception = assertThrows(IOException.class, () -> { + SecurityUtil.createEncryptionContext(conf, mockTableDescriptor, mockFamily, + mockManagedKeyDataCache, null); + }); + assertTrue("Should reference system key cache", + exception.getMessage().contains("SystemKeyCache")); + } + + @Test + public void testWithKeyManagement_NoActiveKey_WithSystemKeyCache() throws IOException { + // Test backwards compatibility: when no active key found but system cache available, should + // work + configBuilder().withKeyManagement(false).apply(conf); + setupManagedKeyDataCache(testTableNamespace, ManagedKeyData.KEY_SPACE_GLOBAL, null); + setupSystemKeyCache(mockManagedKeyData); + when(mockFamily.getEncryptionKey()).thenReturn(null); + + Encryption.Context result = SecurityUtil.createEncryptionContext(conf, mockTableDescriptor, + mockFamily, mockManagedKeyDataCache, mockSystemKeyCache); + + verifyContext(result); + // Should use system key as KEK and generate random DEK + assertEquals(mockManagedKeyData, result.getKEKData()); } @Test @@ -365,8 +409,8 @@ public void testWithKeyManagement_LocalKeyGen_WithUnknownKeyCipher() throws IOEx when(unknownKey.getAlgorithm()).thenReturn(UNKNOWN_CIPHER); when(mockManagedKeyData.getTheKey()).thenReturn(unknownKey); - configBuilder().withKeyManagement(true, true).apply(conf); - setupManagedKeyDataCache(TEST_NAMESPACE, mockManagedKeyData); + configBuilder().withKeyManagement(true).apply(conf); + setupManagedKeyDataCache(testTableNamespace, mockManagedKeyData); assertEncryptionContextThrowsForWrites(RuntimeException.class, "Cipher 'UNKNOWN_CIPHER' is not"); } @@ -377,44 +421,45 @@ public void testWithKeyManagement_LocalKeyGen_WithKeyAlgorithmMismatch() throws when(desKey.getAlgorithm()).thenReturn(DES_CIPHER); when(mockManagedKeyData.getTheKey()).thenReturn(desKey); - configBuilder().withKeyManagement(true, true).apply(conf); - setupManagedKeyDataCache(TEST_NAMESPACE, mockManagedKeyData); + configBuilder().withKeyManagement(true).apply(conf); + setupManagedKeyDataCache(testTableNamespace, mockManagedKeyData); assertEncryptionContextThrowsForWrites(IllegalStateException.class, "Encryption for family 'test-family' configured with type 'AES' but key specifies algorithm 'DES'"); } @Test public void testWithKeyManagement_UseSystemKeyWithNSSpecificActiveKey() throws IOException { - configBuilder().withKeyManagement(true, false).apply(conf); - setupManagedKeyDataCache(TEST_NAMESPACE, mockManagedKeyData); + configBuilder().withKeyManagement(false).apply(conf); + setupManagedKeyDataCache(testTableNamespace, mockManagedKeyData); setupSystemKeyCache(mockManagedKeyData); - Encryption.Context result = SecurityUtil.createEncryptionContext(conf, mockFamily, - mockManagedKeyDataCache, mockSystemKeyCache, TEST_NAMESPACE); + Encryption.Context result = SecurityUtil.createEncryptionContext(conf, mockTableDescriptor, + mockFamily, mockManagedKeyDataCache, mockSystemKeyCache); verifyContext(result); } @Test public void testWithKeyManagement_UseSystemKeyWithoutNSSpecificActiveKey() throws IOException { - configBuilder().withKeyManagement(true, false).apply(conf); - setupManagedKeyDataCache(TEST_NAMESPACE, ManagedKeyData.KEY_SPACE_GLOBAL, mockManagedKeyData); + configBuilder().withKeyManagement(false).apply(conf); + setupManagedKeyDataCache(testTableNamespace, ManagedKeyData.KEY_SPACE_GLOBAL, + mockManagedKeyData); setupSystemKeyCache(mockManagedKeyData); when(mockManagedKeyData.getTheKey()).thenReturn(kekKey); - Encryption.Context result = SecurityUtil.createEncryptionContext(conf, mockFamily, - mockManagedKeyDataCache, mockSystemKeyCache, TEST_NAMESPACE); + Encryption.Context result = SecurityUtil.createEncryptionContext(conf, mockTableDescriptor, + mockFamily, mockManagedKeyDataCache, mockSystemKeyCache); verifyContext(result); } @Test public void testWithoutKeyManagement_WithFamilyProvidedKey() throws Exception { - when(mockFamily.getEncryptionKey()).thenReturn(testWrappedKey); - configBuilder().withKeyManagement(false, false).apply(conf); + byte[] wrappedKey = createRandomWrappedKey(conf); + when(mockFamily.getEncryptionKey()).thenReturn(wrappedKey); - Encryption.Context result = SecurityUtil.createEncryptionContext(conf, mockFamily, - mockManagedKeyDataCache, mockSystemKeyCache, TEST_NAMESPACE); + Encryption.Context result = SecurityUtil.createEncryptionContext(conf, mockTableDescriptor, + mockFamily, mockManagedKeyDataCache, mockSystemKeyCache); verifyContext(result, false); } @@ -426,28 +471,268 @@ public void testWithoutKeyManagement_KeyAlgorithmMismatch() throws Exception { byte[] wrappedDESKey = EncryptionUtil.wrapKey(conf, HBASE_KEY, differentKey); when(mockFamily.getEncryptionKey()).thenReturn(wrappedDESKey); - configBuilder().withKeyManagement(false, false).apply(conf); assertEncryptionContextThrowsForWrites(IllegalStateException.class, "Encryption for family 'test-family' configured with type 'AES' but key specifies algorithm 'DES'"); } @Test - public void testWithoutKeyManagement_WithRandomKeyGeneration() throws IOException { + public void testWithUnavailableCipher() throws IOException { + when(mockFamily.getEncryptionType()).thenReturn(UNKNOWN_CIPHER); + setUpEncryptionConfigWithNullCipher(); + assertEncryptionContextThrowsForWrites(IllegalStateException.class, + "Cipher 'UNKNOWN_CIPHER' is not available"); + } + + // ---- New backwards compatibility test scenarios ---- + + @Test + public void testBackwardsCompatibility_Scenario1_FamilyKeyWithKeyManagement() + throws IOException { + // Scenario 1: Family has encryption key -> use as DEK, latest STK as KEK + when(mockFamily.getEncryptionKey()).thenReturn(testWrappedKey); + configBuilder().withKeyManagement(false).apply(conf); + setupSystemKeyCache(mockManagedKeyData); + when(mockManagedKeyData.getTheKey()).thenReturn(kekKey); + + Encryption.Context result = SecurityUtil.createEncryptionContext(conf, mockTableDescriptor, + mockFamily, mockManagedKeyDataCache, mockSystemKeyCache); + + verifyContext(result); + // Verify that system key is used as KEK + assertEquals(mockManagedKeyData, result.getKEKData()); + } + + @Test + public void testBackwardsCompatibility_Scenario2a_ActiveKeyAsDeK() throws IOException { + // Scenario 2a: Active key exists, local key gen disabled -> use active key as DEK, latest STK + // as KEK + configBuilder().withKeyManagement(false).apply(conf); + setupManagedKeyDataCache(testTableNamespace, mockManagedKeyData); + ManagedKeyData mockSystemKey = mock(ManagedKeyData.class); + when(mockSystemKey.getTheKey()).thenReturn(kekKey); + setupSystemKeyCache(mockSystemKey); when(mockFamily.getEncryptionKey()).thenReturn(null); - configBuilder().withKeyManagement(false, false).apply(conf); - Encryption.Context result = SecurityUtil.createEncryptionContext(conf, mockFamily, - mockManagedKeyDataCache, mockSystemKeyCache, TEST_NAMESPACE); + Encryption.Context result = SecurityUtil.createEncryptionContext(conf, mockTableDescriptor, + mockFamily, mockManagedKeyDataCache, mockSystemKeyCache); + + verifyContext(result); + // Verify that active key is used as DEK and system key as KEK + assertEquals(testKey, result.getKey()); // Active key should be the DEK + assertEquals(mockSystemKey, result.getKEKData()); // System key should be the KEK + } + + @Test + public void testBackwardsCompatibility_Scenario2b_ActiveKeyAsKekWithLocalKeyGen() + throws IOException { + // Scenario 2b: Active key exists, local key gen enabled -> use active key as KEK, generate + // random DEK + configBuilder().withKeyManagement(true).apply(conf); + setupManagedKeyDataCache(testTableNamespace, mockManagedKeyData); + when(mockFamily.getEncryptionKey()).thenReturn(null); + + Encryption.Context result = SecurityUtil.createEncryptionContext(conf, mockTableDescriptor, + mockFamily, mockManagedKeyDataCache, mockSystemKeyCache); + + verifyContext(result); + // Verify that active key is used as KEK and a generated key as DEK + assertNotNull("DEK should be generated", result.getKey()); + assertEquals(mockManagedKeyData, result.getKEKData()); // Active key should be the KEK + } + + @Test + public void testBackwardsCompatibility_Scenario3a_NoActiveKeyGenerateLocalKey() + throws IOException { + // Scenario 3: No active key -> generate random DEK, latest STK as KEK + configBuilder().withKeyManagement(false).apply(conf); + setupManagedKeyDataCache(TEST_NAMESPACE, ManagedKeyData.KEY_SPACE_GLOBAL, null); // No active + // key + setupSystemKeyCache(mockManagedKeyData); + when(mockFamily.getEncryptionKey()).thenReturn(null); + + Encryption.Context result = SecurityUtil.createEncryptionContext(conf, mockTableDescriptor, + mockFamily, mockManagedKeyDataCache, mockSystemKeyCache); + + verifyContext(result); + // Verify that a random key is generated as DEK and system key as KEK + assertNotNull("DEK should be generated", result.getKey()); + assertEquals(mockManagedKeyData, result.getKEKData()); // System key should be the KEK + } + + @Test + public void testWithoutKeyManagement_Scenario3b_WithRandomKeyGeneration() throws IOException { + when(mockFamily.getEncryptionKey()).thenReturn(null); + + Encryption.Context result = SecurityUtil.createEncryptionContext(conf, mockTableDescriptor, + mockFamily, mockManagedKeyDataCache, mockSystemKeyCache); verifyContext(result, false); + // Here system key with a local key gen, so no namespace is set. + assertNull(result.getKeyNamespace()); } @Test - public void testWithUnavailableCipher() throws IOException { - when(mockFamily.getEncryptionType()).thenReturn(UNKNOWN_CIPHER); - setUpEncryptionConfigWithNullCipher(); - assertEncryptionContextThrowsForWrites(IllegalStateException.class, - "Cipher 'UNKNOWN_CIPHER' is not available"); + public void testFallbackRule1_CFKeyNamespaceAttribute() throws IOException { + // Test Rule 1: Column family has KEY_NAMESPACE attribute + String cfKeyNamespace = "cf-specific-namespace"; + when(mockFamily.getEncryptionKeyNamespace()).thenReturn(cfKeyNamespace); + when(mockFamily.getEncryptionKey()).thenReturn(null); + configBuilder().withKeyManagement(false).apply(conf); + + // Mock managed key data cache to return active key only for CF namespace + setupManagedKeyDataCache(cfKeyNamespace, mockManagedKeyData); + setupSystemKeyCache(mockManagedKeyData); + when(mockManagedKeyData.getTheKey()).thenReturn(testKey); + + Encryption.Context result = SecurityUtil.createEncryptionContext(conf, mockTableDescriptor, + mockFamily, mockManagedKeyDataCache, mockSystemKeyCache); + + verifyContext(result); + // Verify that CF-specific namespace was used + assertEquals(cfKeyNamespace, result.getKeyNamespace()); + } + + @Test + public void testFallbackRule2_ConstructedNamespace() throws IOException { + when(mockFamily.getEncryptionKeyNamespace()).thenReturn(null); // No CF namespace + when(mockFamily.getEncryptionKey()).thenReturn(null); + setupManagedKeyDataCache(testTableNamespace, mockManagedKeyData); + configBuilder().withKeyManagement(false).apply(conf); + setupSystemKeyCache(mockManagedKeyData); + + Encryption.Context result = SecurityUtil.createEncryptionContext(conf, mockTableDescriptor, + mockFamily, mockManagedKeyDataCache, mockSystemKeyCache); + + verifyContext(result); + // Verify that constructed namespace was used + assertEquals(testTableNamespace, result.getKeyNamespace()); + } + + @Test + public void testFallbackRule3_TableNameAsNamespace() throws IOException { + // Test Rule 3: Use table name as namespace when CF namespace and constructed namespace fail + when(mockFamily.getEncryptionKeyNamespace()).thenReturn(null); // No CF namespace + when(mockFamily.getEncryptionKey()).thenReturn(null); + configBuilder().withKeyManagement(false).apply(conf); + + String tableName = "test:table"; + when(mockTableDescriptor.getTableName()).thenReturn(TableName.valueOf(tableName)); + + // Mock cache to fail for CF and constructed namespace, succeed for table name + when(mockManagedKeyDataCache.getActiveEntry(eq(ManagedKeyData.KEY_GLOBAL_CUSTODIAN_BYTES), + eq(testTableNamespace))).thenReturn(null); // Constructed namespace fails + when(mockManagedKeyDataCache.getActiveEntry(eq(ManagedKeyData.KEY_GLOBAL_CUSTODIAN_BYTES), + eq(tableName))).thenReturn(mockManagedKeyData); // Table name succeeds + + setupSystemKeyCache(mockManagedKeyData); + when(mockManagedKeyData.getTheKey()).thenReturn(testKey); + + Encryption.Context result = SecurityUtil.createEncryptionContext(conf, mockTableDescriptor, + mockFamily, mockManagedKeyDataCache, mockSystemKeyCache); + + verifyContext(result); + // Verify that table name was used as namespace + assertEquals(tableName, result.getKeyNamespace()); + } + + @Test + public void testFallbackRule4_GlobalNamespace() throws IOException { + // Test Rule 4: Fall back to global namespace when all other rules fail + when(mockFamily.getEncryptionKeyNamespace()).thenReturn(null); // No CF namespace + when(mockFamily.getEncryptionKey()).thenReturn(null); + configBuilder().withKeyManagement(false).apply(conf); + + String tableName = "test:table"; + when(mockTableDescriptor.getTableName()).thenReturn(TableName.valueOf(tableName)); + + // Mock cache to fail for all specific namespaces, succeed only for global + when(mockManagedKeyDataCache.getActiveEntry(eq(ManagedKeyData.KEY_GLOBAL_CUSTODIAN_BYTES), + eq(testTableNamespace))).thenReturn(null); // Constructed namespace fails + when(mockManagedKeyDataCache.getActiveEntry(eq(ManagedKeyData.KEY_GLOBAL_CUSTODIAN_BYTES), + eq(tableName))).thenReturn(null); // Table name fails + when(mockManagedKeyDataCache.getActiveEntry(eq(ManagedKeyData.KEY_GLOBAL_CUSTODIAN_BYTES), + eq(ManagedKeyData.KEY_SPACE_GLOBAL))).thenReturn(mockManagedKeyData); // Global succeeds + + setupSystemKeyCache(mockManagedKeyData); + when(mockManagedKeyData.getTheKey()).thenReturn(testKey); + + Encryption.Context result = SecurityUtil.createEncryptionContext(conf, mockTableDescriptor, + mockFamily, mockManagedKeyDataCache, mockSystemKeyCache); + + verifyContext(result); + // Verify that global namespace was used + assertEquals(ManagedKeyData.KEY_SPACE_GLOBAL, result.getKeyNamespace()); + } + + @Test + public void testFallbackRuleOrder() throws IOException { + // Test that the rules are tried in the correct order + String cfKeyNamespace = "cf-namespace"; + String tableName = "test:table"; + + when(mockFamily.getEncryptionKeyNamespace()).thenReturn(cfKeyNamespace); + when(mockFamily.getEncryptionKey()).thenReturn(null); + when(mockTableDescriptor.getTableName()).thenReturn(TableName.valueOf(tableName)); + configBuilder().withKeyManagement(false).apply(conf); + + // Set up mocks so that CF namespace fails but table name would succeed + when(mockManagedKeyDataCache.getActiveEntry(eq(ManagedKeyData.KEY_GLOBAL_CUSTODIAN_BYTES), + eq(cfKeyNamespace))).thenReturn(null); // CF namespace fails + when(mockManagedKeyDataCache.getActiveEntry(eq(ManagedKeyData.KEY_GLOBAL_CUSTODIAN_BYTES), + eq(testTableNamespace))).thenReturn(mockManagedKeyData); // Constructed namespace succeeds + when(mockManagedKeyDataCache.getActiveEntry(eq(ManagedKeyData.KEY_GLOBAL_CUSTODIAN_BYTES), + eq(tableName))).thenReturn(mockManagedKeyData); // Table name would also succeed + + setupSystemKeyCache(mockManagedKeyData); + when(mockManagedKeyData.getTheKey()).thenReturn(testKey); + + Encryption.Context result = SecurityUtil.createEncryptionContext(conf, mockTableDescriptor, + mockFamily, mockManagedKeyDataCache, mockSystemKeyCache); + + verifyContext(result); + // Verify that constructed namespace was used (Rule 2), not table name (Rule 3) + assertEquals(testTableNamespace, result.getKeyNamespace()); + } + + @Test + public void testBackwardsCompatibility_Scenario1_FamilyKeyWithoutKeyManagement() + throws IOException { + // Scenario 1 variation: Family has encryption key but key management disabled -> use as DEK, + // no KEK + byte[] wrappedKey = createRandomWrappedKey(conf); + when(mockFamily.getEncryptionKey()).thenReturn(wrappedKey); + + Encryption.Context result = SecurityUtil.createEncryptionContext(conf, mockTableDescriptor, + mockFamily, mockManagedKeyDataCache, mockSystemKeyCache); + + verifyContext(result, false); // No key management, so no KEK data + } + + @Test + public void testWithKeyManagement_FamilyKey_UnwrapKeyException() throws Exception { + // Test for KeyException->IOException wrapping when family has key bytes with key management + // enabled + // This covers the exception block at lines 103-105 in SecurityUtil.java + + // Create a properly wrapped key first, then corrupt it to cause unwrapping failure + Key wrongKek = new SecretKeySpec("bad-kek-16-bytes".getBytes(), AES_CIPHER); // Exactly 16 + // bytes + byte[] validWrappedKey = EncryptionUtil.wrapKey(conf, null, testKey, wrongKek); + + when(mockFamily.getEncryptionKey()).thenReturn(validWrappedKey); + configBuilder().withKeyManagement(false).apply(conf); + setupSystemKeyCache(mockManagedKeyData); + when(mockManagedKeyData.getTheKey()).thenReturn(kekKey); // Different KEK for unwrapping + + IOException exception = assertThrows(IOException.class, () -> { + SecurityUtil.createEncryptionContext(conf, mockTableDescriptor, mockFamily, + mockManagedKeyDataCache, mockSystemKeyCache); + }); + + // The IOException should wrap a KeyException from the unwrapping process + assertNotNull("Exception should have a cause", exception.getCause()); + assertTrue("Exception cause should be a KeyException", + exception.getCause() instanceof KeyException); } // Tests for the second createEncryptionContext method (for reading files) @@ -473,9 +758,42 @@ public static class TestCreateEncryptionContext_ForReads extends TestSecurityUti HBaseClassTestRule.forClass(TestCreateEncryptionContext_ForReads.class); @Test - public void testWithKEKMetadata() throws Exception { - setupTrailerMocks(testWrappedKey, TEST_KEK_METADATA, TEST_KEK_CHECKSUM, TEST_NAMESPACE); - setupManagedKeyDataCacheEntry(TEST_NAMESPACE, TEST_KEK_METADATA, testWrappedKey, + public void testWithKEKMetadata_STKLookupFirstThenManagedKey() throws Exception { + // Test new logic: STK lookup happens first, then metadata lookup if STK fails + // Set up scenario where both checksum and metadata are available + setupTrailerMocks(testWrappedKey, TEST_KEK_METADATA, TEST_KEK_CHECKSUM, null); + configBuilder().withKeyManagement(false).apply(conf); + + // STK lookup should succeed and be used (first priority) + ManagedKeyData stkKeyData = mock(ManagedKeyData.class); + when(stkKeyData.getTheKey()).thenReturn(kekKey); + setupSystemKeyCache(TEST_KEK_CHECKSUM, stkKeyData); + + // Also set up managed key cache (but it shouldn't be used since STK succeeds) + setupManagedKeyDataCacheEntry(testTableNamespace, TEST_KEK_METADATA, testWrappedKey, + mockManagedKeyData); + when(mockManagedKeyData.getTheKey()) + .thenThrow(new RuntimeException("This should not be called")); + + Encryption.Context result = SecurityUtil.createEncryptionContext(conf, testPath, mockTrailer, + mockManagedKeyDataCache, mockSystemKeyCache); + + verifyContext(result); + // Should use STK data, not managed key data + assertEquals(stkKeyData, result.getKEKData()); + } + + @Test + public void testWithKEKMetadata_STKFailsThenManagedKeySucceeds() throws Exception { + // Test fallback: STK lookup fails, metadata lookup succeeds + setupTrailerMocks(testWrappedKey, TEST_KEK_METADATA, TEST_KEK_CHECKSUM, testTableNamespace); + configBuilder().withKeyManagement(false).apply(conf); + + // STK lookup should fail (returns null) + when(mockSystemKeyCache.getSystemKeyByChecksum(TEST_KEK_CHECKSUM)).thenReturn(null); + + // Managed key lookup should succeed + setupManagedKeyDataCacheEntry(testTableNamespace, TEST_KEK_METADATA, testWrappedKey, mockManagedKeyData); when(mockManagedKeyData.getTheKey()).thenReturn(kekKey); @@ -483,17 +801,27 @@ public void testWithKEKMetadata() throws Exception { mockManagedKeyDataCache, mockSystemKeyCache); verifyContext(result); + // Should use managed key data since STK failed + assertEquals(mockManagedKeyData, result.getKEKData()); } @Test - public void testWithKeyManagement_KEKMetadataFailure() throws IOException, KeyException { + public void testWithKeyManagement_KEKMetadataAndChecksumFailure() + throws IOException, KeyException { + // Test scenario where both STK lookup and managed key lookup fail byte[] keyBytes = "test-encrypted-key".getBytes(); String kekMetadata = "test-kek-metadata"; + configBuilder().withKeyManagement(false).apply(conf); when(mockTrailer.getEncryptionKey()).thenReturn(keyBytes); when(mockTrailer.getKEKMetadata()).thenReturn(kekMetadata); + when(mockTrailer.getKEKChecksum()).thenReturn(TEST_KEK_CHECKSUM); when(mockTrailer.getKeyNamespace()).thenReturn("test-namespace"); + // STK lookup should fail + when(mockSystemKeyCache.getSystemKeyByChecksum(TEST_KEK_CHECKSUM)).thenReturn(null); + + // Managed key lookup should also fail when(mockManagedKeyDataCache.getEntry(eq(ManagedKeyData.KEY_GLOBAL_CUSTODIAN_BYTES), eq("test-namespace"), eq(kekMetadata), eq(keyBytes))) .thenThrow(new IOException("Key not found")); @@ -503,13 +831,16 @@ public void testWithKeyManagement_KEKMetadataFailure() throws IOException, KeyEx mockSystemKeyCache); }); - assertTrue(exception.getMessage().contains("Failed to get key data")); + assertTrue( + exception.getMessage().contains("Failed to get key data for KEK metadata: " + kekMetadata)); + assertTrue(exception.getCause().getMessage().contains("Key not found")); } @Test public void testWithKeyManagement_UseSystemKey() throws IOException { - setupTrailerMocks(testWrappedKey, null, TEST_KEK_CHECKSUM, TEST_NAMESPACE); - configBuilder().withKeyManagement(true, false).apply(conf); + // Test STK lookup by checksum (first priority in new logic) + setupTrailerMocks(testWrappedKey, null, TEST_KEK_CHECKSUM, null); + configBuilder().withKeyManagement(false).apply(conf); setupSystemKeyCache(TEST_KEK_CHECKSUM, mockManagedKeyData); when(mockManagedKeyData.getTheKey()).thenReturn(kekKey); @@ -517,38 +848,53 @@ public void testWithKeyManagement_UseSystemKey() throws IOException { mockManagedKeyDataCache, mockSystemKeyCache); verifyContext(result); + assertEquals(mockManagedKeyData, result.getKEKData()); } @Test - public void testWithKeyManagement_SystemKeyNotFound() throws IOException { + public void testBackwardsCompatibility_WithKeyManagement_LatestSystemKeyNotFound() + throws IOException { + // Test when both STK lookup by checksum fails and latest system key is null byte[] keyBytes = "test-encrypted-key".getBytes(); - long kekChecksum = 12345L; when(mockTrailer.getEncryptionKey()).thenReturn(keyBytes); - when(mockTrailer.getKEKMetadata()).thenReturn(null); - when(mockTrailer.getKEKChecksum()).thenReturn(kekChecksum); - when(mockTrailer.getKeyNamespace()).thenReturn("test-namespace"); // Enable key management conf.setBoolean(HConstants.CRYPTO_MANAGED_KEYS_ENABLED_CONF_KEY, true); - when(mockSystemKeyCache.getSystemKeyByChecksum(kekChecksum)).thenReturn(null); + // Both checksum lookup and latest system key lookup should fail + when(mockSystemKeyCache.getLatestSystemKey()).thenReturn(null); IOException exception = assertThrows(IOException.class, () -> { SecurityUtil.createEncryptionContext(conf, testPath, mockTrailer, mockManagedKeyDataCache, mockSystemKeyCache); }); - assertTrue(exception.getMessage().contains("Failed to get system key")); + assertTrue(exception.getMessage().contains("Failed to get latest system key")); + } + + @Test + public void testBackwardsCompatibility_FallbackToLatestSystemKey() throws IOException { + // Test fallback to latest system key when both checksum and metadata are unavailable + setupTrailerMocks(testWrappedKey, null, 0L, TEST_NAMESPACE); // No checksum, no metadata + configBuilder().withKeyManagement(false).apply(conf); + + ManagedKeyData latestSystemKey = mock(ManagedKeyData.class); + when(latestSystemKey.getTheKey()).thenReturn(kekKey); + when(mockSystemKeyCache.getLatestSystemKey()).thenReturn(latestSystemKey); + + Encryption.Context result = SecurityUtil.createEncryptionContext(conf, testPath, mockTrailer, + mockManagedKeyDataCache, mockSystemKeyCache); + + verifyContext(result); + assertEquals(latestSystemKey, result.getKEKData()); } @Test public void testWithoutKeyManagemntEnabled() throws IOException { - when(mockTrailer.getEncryptionKey()).thenReturn(testWrappedKey); + byte[] wrappedKey = createRandomWrappedKey(conf); + when(mockTrailer.getEncryptionKey()).thenReturn(wrappedKey); when(mockTrailer.getKEKMetadata()).thenReturn(null); - when(mockTrailer.getKeyNamespace()).thenReturn(TEST_NAMESPACE); - configBuilder().withKeyManagement(false, false).apply(conf); - // TODO: Get the key provider to return kek when getKeys() is called. Encryption.Context result = SecurityUtil.createEncryptionContext(conf, testPath, mockTrailer, mockManagedKeyDataCache, mockSystemKeyCache); @@ -556,13 +902,24 @@ public void testWithoutKeyManagemntEnabled() throws IOException { verifyContext(result, false); } + @Test + public void testKeyManagementBackwardsCompatibility() throws Exception { + when(mockTrailer.getEncryptionKey()).thenReturn(testWrappedKey); + when(mockSystemKeyCache.getLatestSystemKey()).thenReturn(mockManagedKeyData); + when(mockManagedKeyData.getTheKey()).thenReturn(kekKey); + configBuilder().withKeyManagement(false).apply(conf); + + Encryption.Context result = SecurityUtil.createEncryptionContext(conf, testPath, mockTrailer, + mockManagedKeyDataCache, mockSystemKeyCache); + + verifyContext(result, true); + } + @Test public void testWithoutKeyManagement_UnwrapFailure() throws IOException { byte[] invalidKeyBytes = INVALID_KEY_DATA.getBytes(); when(mockTrailer.getEncryptionKey()).thenReturn(invalidKeyBytes); when(mockTrailer.getKEKMetadata()).thenReturn(null); - when(mockTrailer.getKeyNamespace()).thenReturn(TEST_NAMESPACE); - configBuilder().withKeyManagement(false, false).apply(conf); Exception exception = assertThrows(Exception.class, () -> { SecurityUtil.createEncryptionContext(conf, testPath, mockTrailer, mockManagedKeyDataCache, @@ -579,11 +936,10 @@ public void testCreateEncryptionContext_WithoutKeyManagement_UnavailableCipher() throws Exception { // Create a DES key and wrap it first with working configuration Key desKey = new SecretKeySpec("test-key-16-byte".getBytes(), "DES"); - byte[] wrappedDESKey = EncryptionUtil.wrapKey(conf, "hbase", desKey); + byte[] wrappedDESKey = EncryptionUtil.wrapKey(conf, HBASE_KEY, desKey); when(mockTrailer.getEncryptionKey()).thenReturn(wrappedDESKey); when(mockTrailer.getKEKMetadata()).thenReturn(null); - when(mockTrailer.getKeyNamespace()).thenReturn("test-namespace"); // Disable key management and use null cipher provider conf.setBoolean(HConstants.CRYPTO_MANAGED_KEYS_ENABLED_CONF_KEY, false); @@ -634,7 +990,8 @@ public void testCreateEncryptionContext_WithKeyManagement_NullSystemKeyCache() null); }); - assertTrue(exception.getMessage().contains("SystemKeyCache is null")); + assertTrue(exception.getMessage() + .contains("SystemKeyCache can't be null when using key management feature")); } } @@ -654,18 +1011,14 @@ public static Collection data() { return Arrays.asList(new Object[][] { { true }, { false }, }); } - @Test - public void test() throws IOException { - } - @Test public void testWithDEK() throws IOException, KeyException { - // This test is challenging because we need to create a scenario where unwrapping fails - // with either KeyException or IOException. We'll create invalid wrapped data. - byte[] invalidKeyBytes = INVALID_WRAPPED_KEY_DATA.getBytes(); + byte[] wrappedKey = createRandomWrappedKey(conf); + MockAesKeyProvider keyProvider = (MockAesKeyProvider) Encryption.getKeyProvider(conf); + keyProvider.clearKeys(); // Let a new key be instantiated and cause a unwrap failure. - setupTrailerMocks(invalidKeyBytes, TEST_KEK_METADATA, TEST_KEK_CHECKSUM, TEST_NAMESPACE); - setupManagedKeyDataCacheEntry(TEST_NAMESPACE, TEST_KEK_METADATA, invalidKeyBytes, + setupTrailerMocks(wrappedKey, null, 0L, null); + setupManagedKeyDataCacheEntry(TEST_NAMESPACE, TEST_KEK_METADATA, wrappedKey, mockManagedKeyData); IOException exception = assertThrows(IOException.class, () -> { @@ -673,8 +1026,7 @@ public void testWithDEK() throws IOException, KeyException { mockSystemKeyCache); }); - assertTrue(exception.getMessage().contains("Failed to unwrap key with KEK checksum: " - + TEST_KEK_CHECKSUM + ", metadata: " + TEST_KEK_METADATA)); + assertTrue(exception.getMessage().contains("Key was not successfully unwrapped")); // The root cause should be some kind of parsing/unwrapping exception assertNotNull(exception.getCause()); } @@ -684,8 +1036,8 @@ public void testWithSystemKey() throws IOException { // Use invalid key bytes to trigger unwrapping failure byte[] invalidKeyBytes = INVALID_SYSTEM_KEY_DATA.getBytes(); - setupTrailerMocks(invalidKeyBytes, null, TEST_KEK_CHECKSUM, TEST_NAMESPACE); - configBuilder().withKeyManagement(true, false).apply(conf); + setupTrailerMocks(invalidKeyBytes, null, TEST_KEK_CHECKSUM, null); + configBuilder().withKeyManagement(false).apply(conf); setupSystemKeyCache(TEST_KEK_CHECKSUM, mockManagedKeyData); IOException exception = assertThrows(IOException.class, () -> { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestEncryptionTest.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestEncryptionTest.java index f0cc2febd6e8..7b67b838659b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestEncryptionTest.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestEncryptionTest.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hbase.util; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.IOException; @@ -30,7 +31,9 @@ import org.apache.hadoop.hbase.io.crypto.DefaultCipherProvider; import org.apache.hadoop.hbase.io.crypto.Encryption; import org.apache.hadoop.hbase.io.crypto.KeyProvider; +import org.apache.hadoop.hbase.io.crypto.ManagedKeyStoreKeyProvider; import org.apache.hadoop.hbase.io.crypto.MockAesKeyProvider; +import org.apache.hadoop.hbase.io.crypto.MockManagedKeyProvider; import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.junit.ClassRule; @@ -130,6 +133,71 @@ public void testTestEnabledWhenCryptoIsExplicitlyDisabled() throws Exception { EncryptionTest.testEncryption(conf, algorithm, null); } + // Utility methods for configuration setup + private Configuration createManagedKeyProviderConfig() { + Configuration conf = HBaseConfiguration.create(); + conf.setBoolean(HConstants.CRYPTO_MANAGED_KEYS_ENABLED_CONF_KEY, true); + conf.set(HConstants.CRYPTO_MANAGED_KEYPROVIDER_CONF_KEY, + MockManagedKeyProvider.class.getName()); + return conf; + } + + @Test + public void testManagedKeyProvider() throws Exception { + Configuration conf = createManagedKeyProviderConfig(); + EncryptionTest.testKeyProvider(conf); + assertTrue("Managed provider should be cached", EncryptionTest.keyProviderResults + .containsKey(conf.get(HConstants.CRYPTO_MANAGED_KEYPROVIDER_CONF_KEY))); + } + + @Test(expected = IOException.class) + public void testBadManagedKeyProvider() throws Exception { + Configuration conf = HBaseConfiguration.create(); + conf.setBoolean(HConstants.CRYPTO_MANAGED_KEYS_ENABLED_CONF_KEY, true); + conf.set(HConstants.CRYPTO_MANAGED_KEYPROVIDER_CONF_KEY, + FailingManagedKeyProvider.class.getName()); + EncryptionTest.testKeyProvider(conf); + fail("Instantiation of bad managed key provider should have failed check"); + } + + @Test + public void testEncryptionWithManagedKeyProvider() throws Exception { + Configuration conf = createManagedKeyProviderConfig(); + String algorithm = conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES); + EncryptionTest.testEncryption(conf, algorithm, null); + assertTrue("Managed provider should be cached", EncryptionTest.keyProviderResults + .containsKey(conf.get(HConstants.CRYPTO_MANAGED_KEYPROVIDER_CONF_KEY))); + } + + @Test(expected = IOException.class) + public void testUnknownCipherWithManagedKeyProvider() throws Exception { + Configuration conf = createManagedKeyProviderConfig(); + EncryptionTest.testEncryption(conf, "foobar", null); + fail("Test for bogus cipher should have failed with managed key provider"); + } + + @Test(expected = IOException.class) + public void testManagedKeyProviderWhenCryptoIsExplicitlyDisabled() throws Exception { + Configuration conf = createManagedKeyProviderConfig(); + String algorithm = conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES); + conf.setBoolean(Encryption.CRYPTO_ENABLED_CONF_KEY, false); + EncryptionTest.testEncryption(conf, algorithm, null); + assertTrue("Managed provider should be cached", EncryptionTest.keyProviderResults + .containsKey(conf.get(HConstants.CRYPTO_MANAGED_KEYPROVIDER_CONF_KEY))); + } + + @Test(expected = IOException.class) + public void testManagedKeyProviderWithKeyManagementDisabled() throws Exception { + Configuration conf = HBaseConfiguration.create(); + conf.setBoolean(HConstants.CRYPTO_MANAGED_KEYS_ENABLED_CONF_KEY, false); + // This should cause issues since we're trying to use managed provider without enabling key + // management + conf.set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, ManagedKeyStoreKeyProvider.class.getName()); + + EncryptionTest.testKeyProvider(conf); + fail("Should have failed when using managed provider with key management disabled"); + } + public static class FailingKeyProvider implements KeyProvider { @Override @@ -181,4 +249,12 @@ public Cipher getCipher(String name) { } } + + // Helper class for testing failing managed key provider + public static class FailingManagedKeyProvider extends MockManagedKeyProvider { + @Override + public void initConfig(Configuration conf, String params) { + throw new RuntimeException("BAD MANAGED PROVIDER!"); + } + } } diff --git a/hbase-shell/src/main/ruby/hbase/admin.rb b/hbase-shell/src/main/ruby/hbase/admin.rb index 93cc312338c9..2b1d29e7849e 100644 --- a/hbase-shell/src/main/ruby/hbase/admin.rb +++ b/hbase-shell/src/main/ruby/hbase/admin.rb @@ -1223,6 +1223,10 @@ def cfd(arg, tdb) cfdb.setEncryptionKey(org.apache.hadoop.hbase.security.EncryptionUtil.wrapKey(@conf, key, algorithm)) end + if arg.include?(ColumnFamilyDescriptorBuilder::ENCRYPTION_KEY_NAMESPACE) + cfdb.setEncryptionKeyNamespace(arg.delete( + ColumnFamilyDescriptorBuilder::ENCRYPTION_KEY_NAMESPACE)) + end end if arg.include?(ColumnFamilyDescriptorBuilder::COMPRESSION_COMPACT) compression = arg.delete(ColumnFamilyDescriptorBuilder::COMPRESSION_COMPACT).upcase.to_sym diff --git a/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestKeymetaAdminShell.java b/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestKeymetaAdminShell.java index b67fbc69f3c7..8315d05f3feb 100644 --- a/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestKeymetaAdminShell.java +++ b/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestKeymetaAdminShell.java @@ -17,7 +17,6 @@ */ package org.apache.hadoop.hbase.client; -import java.util.Base64; import java.util.HashMap; import java.util.Map; import java.util.Properties; @@ -50,47 +49,64 @@ public class TestKeymetaAdminShell extends ManagedKeyTestBase implements RubyShe @Before public void setUp() throws Exception { final Configuration conf = TEST_UTIL.getConfiguration(); - conf.set("zookeeper.session.timeout", "6000000"); - conf.set("hbase.rpc.timeout", "6000000"); - conf.set("hbase.rpc.read.timeout", "6000000"); - conf.set("hbase.rpc.write.timeout", "6000000"); - conf.set("hbase.client.operation.timeout", "6000000"); - conf.set("hbase.client.scanner.timeout.period", "6000000"); - conf.set("hbase.ipc.client.socket.timeout.connect", "6000000"); - conf.set("hbase.ipc.client.socket.timeout.read", "6000000"); - conf.set("hbase.ipc.client.socket.timeout.write", "6000000"); - conf.set("hbase.master.start.timeout.localHBaseCluster", "6000000"); - conf.set("hbase.master.init.timeout.localHBaseCluster", "6000000"); - conf.set("hbase.client.sync.wait.timeout.msec", "6000000"); - Map cust2key = new HashMap<>(); - Map cust2alias = new HashMap<>(); + // Enable to be able to debug without timing out. + // conf.set("zookeeper.session.timeout", "6000000"); + // conf.set("hbase.rpc.timeout", "6000000"); + // conf.set("hbase.rpc.read.timeout", "6000000"); + // conf.set("hbase.rpc.write.timeout", "6000000"); + // conf.set("hbase.client.operation.timeout", "6000000"); + // conf.set("hbase.client.scanner.timeout.period", "6000000"); + // conf.set("hbase.ipc.client.socket.timeout.connect", "6000000"); + // conf.set("hbase.ipc.client.socket.timeout.read", "6000000"); + // conf.set("hbase.ipc.client.socket.timeout.write", "6000000"); + // conf.set("hbase.master.start.timeout.localHBaseCluster", "6000000"); + // conf.set("hbase.master.init.timeout.localHBaseCluster", "6000000"); + // conf.set("hbase.client.sync.wait.timeout.msec", "6000000"); + // conf.set("hbase.client.retries.number", "1000"); + Map cust_to_key = new HashMap<>(); + Map cust_to_alias = new HashMap<>(); String clusterId = UUID.randomUUID().toString(); String SYSTEM_KEY_ALIAS = "system-key-alias"; String CUST1 = "cust1"; String CUST1_ALIAS = "cust1-alias"; + String CF_NAMESPACE = "test_table/f"; String GLOB_CUST_ALIAS = "glob-cust-alias"; - String providerParams = KeymetaTestUtils.setupTestKeyStore(TEST_UTIL, true, true, store -> { - Properties p = new Properties(); - try { - KeymetaTestUtils.addEntry(conf, 128, store, CUST1_ALIAS, CUST1, true, cust2key, cust2alias, - p); - KeymetaTestUtils.addEntry(conf, 128, store, GLOB_CUST_ALIAS, "*", true, cust2key, - cust2alias, p); - KeymetaTestUtils.addEntry(conf, 128, store, SYSTEM_KEY_ALIAS, clusterId, true, cust2key, - cust2alias, p); - } catch (Exception e) { - throw new RuntimeException(e); - } - return p; - }); - // byte[] systemKey = cust2key.get(new Bytes(clusterId.getBytes())).get(); - conf.set(HConstants.CRYPTO_MANAGED_KEY_STORE_SYSTEM_KEY_NAME_CONF_KEY, SYSTEM_KEY_ALIAS); - conf.set(HConstants.CRYPTO_KEYPROVIDER_PARAMETERS_KEY, providerParams); + String CUSTOM_NAMESPACE = "test_namespace"; + String CUSTOM_NAMESPACE_ALIAS = "custom-namespace-alias"; + String CUSTOM_GLOBAL_NAMESPACE = "test_global_namespace"; + String CUSTOM_GLOBAL_NAMESPACE_ALIAS = "custom-global-namespace-alias"; + if (isWithKeyManagement()) { + String providerParams = KeymetaTestUtils.setupTestKeyStore(TEST_UTIL, true, true, store -> { + Properties p = new Properties(); + try { + KeymetaTestUtils.addEntry(conf, 128, store, CUST1_ALIAS, CUST1, true, cust_to_key, + cust_to_alias, p); + KeymetaTestUtils.addEntry(conf, 128, store, CUST1_ALIAS, CUST1, true, cust_to_key, + cust_to_alias, p, CF_NAMESPACE); + KeymetaTestUtils.addEntry(conf, 128, store, GLOB_CUST_ALIAS, "*", true, cust_to_key, + cust_to_alias, p); + KeymetaTestUtils.addEntry(conf, 128, store, SYSTEM_KEY_ALIAS, clusterId, true, + cust_to_key, cust_to_alias, p); + KeymetaTestUtils.addEntry(conf, 128, store, CUSTOM_NAMESPACE_ALIAS, CUST1, true, + cust_to_key, cust_to_alias, p, CUSTOM_NAMESPACE); + KeymetaTestUtils.addEntry(conf, 128, store, CUSTOM_GLOBAL_NAMESPACE_ALIAS, "*", true, + cust_to_key, cust_to_alias, p, CUSTOM_GLOBAL_NAMESPACE); + } catch (Exception e) { + throw new RuntimeException(e); + } + return p; + }); + // byte[] systemKey = cust2key.get(new Bytes(clusterId.getBytes())).get(); + conf.set(HConstants.CRYPTO_MANAGED_KEY_STORE_SYSTEM_KEY_NAME_CONF_KEY, SYSTEM_KEY_ALIAS); + conf.set(HConstants.CRYPTO_MANAGED_KEYPROVIDER_PARAMETERS_KEY, providerParams); + } RubyShellTest.setUpConfig(this); super.setUp(); RubyShellTest.setUpJRubyRuntime(this); RubyShellTest.doTestSetup(this); + addCustodianRubyEnvVars(jruby, "GLOB_CUST", "*"); addCustodianRubyEnvVars(jruby, "CUST1", CUST1); + jruby.put("$TEST", this); } @Override @@ -122,6 +138,6 @@ public static void addCustodianRubyEnvVars(ScriptingContainer jruby, String cust String custodian) { jruby.put("$" + custId, custodian); jruby.put("$" + custId + "_ALIAS", custodian + "-alias"); - jruby.put("$" + custId + "_ENCODED", Base64.getEncoder().encodeToString(custodian.getBytes())); + jruby.put("$" + custId + "_ENCODED", ManagedKeyProvider.encodeToStr(custodian.getBytes())); } } diff --git a/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestKeymetaMigration.java b/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestKeymetaMigration.java new file mode 100644 index 000000000000..efe124989e56 --- /dev/null +++ b/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestKeymetaMigration.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.testclassification.ClientTests; +import org.apache.hadoop.hbase.testclassification.IntegrationTests; +import org.junit.ClassRule; +import org.junit.experimental.categories.Category; + +@Category({ ClientTests.class, IntegrationTests.class }) +public class TestKeymetaMigration extends TestKeymetaAdminShell { + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestKeymetaMigration.class); + + @Override + public String getSuitePattern() { + return "**/*_keymeta_migration_test.rb"; + } + + @Override + protected boolean isWithKeyManagement() { + return false; + } + + @Override + protected boolean isWithMiniClusterStart() { + return false; + } + + @Override + protected TableName getSystemTableNameToWaitFor() { + return TableName.META_TABLE_NAME; + } +} diff --git a/hbase-shell/src/test/ruby/shell/admin_keymeta_test.rb b/hbase-shell/src/test/ruby/shell/admin_keymeta_test.rb index c1108d0fc7d1..9f3048ab5991 100644 --- a/hbase-shell/src/test/ruby/shell/admin_keymeta_test.rb +++ b/hbase-shell/src/test/ruby/shell/admin_keymeta_test.rb @@ -35,27 +35,32 @@ def setup end define_test 'Test enable key management' do - cust_and_namespace = "#{$CUST1_ENCODED}:*" + test_key_management($CUST1_ENCODED, '*') + test_key_management($CUST1_ENCODED, 'test_table/f') + test_key_management($CUST1_ENCODED, 'test_namespace') + test_key_management($GLOB_CUST_ENCODED, '*') + + puts "Testing that cluster can be restarted when key management is enabled" + $TEST.restartMiniCluster() + puts "Cluster restarted, testing key management again" + setup_hbase + test_key_management($GLOB_CUST_ENCODED, '*') + puts "Key management test complete" + end + + def test_key_management(cust, namespace) # Repeat the enable twice in a loop and ensure multiple enables succeed and return the # same output. 2.times do |i| + cust_and_namespace = "#{cust}:#{namespace}" output = capture_stdout { @shell.command('enable_key_management', cust_and_namespace) } - puts "enable_key_management #{i} output: #{output}" - assert(output.include?("#{$CUST1_ENCODED} * ACTIVE")) + puts "enable_key_management output: #{output}" + assert(output.include?("#{cust} #{namespace} ACTIVE")) + output = capture_stdout { @shell.command('show_key_status', cust_and_namespace) } + puts "show_key_status output: #{output}" + assert(output.include?("#{cust} #{namespace} ACTIVE")) + assert(output.include?('1 row(s)')) end - output = capture_stdout { @shell.command('show_key_status', cust_and_namespace) } - puts "show_key_status output: #{output}" - assert(output.include?("#{$CUST1_ENCODED} * ACTIVE")) - - # The ManagedKeyStoreKeyProvider doesn't support specific namespaces, so it will return the - # global key. - cust_and_namespace = "#{$CUST1_ENCODED}:test_table/f" - output = capture_stdout { @shell.command('enable_key_management', cust_and_namespace) } - puts "enable_key_management output: #{output}" - assert(output.include?("#{$CUST1_ENCODED} * ACTIVE")) - output = capture_stdout { @shell.command('show_key_status', cust_and_namespace) } - puts "show_key_status output: #{output}" - assert(output.include?('0 row(s)')) end end end diff --git a/hbase-shell/src/test/ruby/shell/encrypted_table_keymeta_test.rb b/hbase-shell/src/test/ruby/shell/encrypted_table_keymeta_test.rb index be52a2524e4d..2562a64779e0 100644 --- a/hbase-shell/src/test/ruby/shell/encrypted_table_keymeta_test.rb +++ b/hbase-shell/src/test/ruby/shell/encrypted_table_keymeta_test.rb @@ -31,6 +31,7 @@ java_import org.apache.hadoop.hbase.HConstants java_import org.apache.hadoop.hbase.client.Get java_import org.apache.hadoop.hbase.io.crypto.Encryption +java_import org.apache.hadoop.hbase.io.crypto.ManagedKeyProvider java_import org.apache.hadoop.hbase.io.crypto.MockManagedKeyProvider java_import org.apache.hadoop.hbase.io.hfile.CorruptHFileException java_import org.apache.hadoop.hbase.io.hfile.FixedFileTrailer @@ -45,14 +46,29 @@ class EncryptedTableKeymetaTest < Test::Unit::TestCase def setup setup_hbase - @test_table = 'enctest' + @test_table = 'enctest'+Time.now.to_i.to_s @connection = $TEST_CLUSTER.connection end define_test 'Test table put/get with encryption' do - cust_and_namespace = "#{$CUST1_ENCODED}:*" - @shell.command(:enable_key_management, cust_and_namespace) - @shell.command(:create, @test_table, { 'NAME' => 'f', 'ENCRYPTION' => 'AES' }) + # Custodian is currently not supported, so this will end up falling back to local key + # generation. + test_table_put_get_with_encryption($CUST1_ENCODED, '*', + { 'NAME' => 'f', 'ENCRYPTION' => 'AES' }, true) + end + + define_test 'Test table with custom namespace attribute in Column Family' do + custom_namespace = "test_global_namespace" + test_table_put_get_with_encryption($GLOB_CUST_ENCODED, custom_namespace, + { 'NAME' => 'f', 'ENCRYPTION' => 'AES', 'ENCRYPTION_KEY_NAMESPACE' => custom_namespace }, + false) + end + + def test_table_put_get_with_encryption(cust, namespace, table_attrs, fallback_scenario) + cust_and_namespace = "#{cust}:#{namespace}" + output = capture_stdout { @shell.command('enable_key_management', cust_and_namespace) } + assert(output.include?("#{cust} #{namespace} ACTIVE")) + @shell.command(:create, @test_table, table_attrs) test_table = table(@test_table) test_table.put('1', 'f:a', '2') puts "Added a row, now flushing table #{@test_table}" @@ -72,6 +88,20 @@ def setup assert_not_nil(hfile_info) live_trailer = hfile_info.getTrailer assert_trailer(live_trailer) + assert_equal(namespace, live_trailer.getKeyNamespace()) + + # When active key is supposed to be used, we can valiate the key bytes in the context against + # the actual key from provider. + if !fallback_scenario + encryption_context = hfile_info.getHFileContext().getEncryptionContext() + assert_not_nil(encryption_context) + assert_not_nil(encryption_context.getKeyBytes()) + key_provider = Encryption.getManagedKeyProvider($TEST_CLUSTER.getConfiguration) + key_data = key_provider.getManagedKey(ManagedKeyProvider.decodeToBytes(cust), namespace) + assert_not_nil(key_data) + assert_equal(namespace, key_data.getKeyNamespace()) + assert_equal(key_data.getTheKey().getEncoded(), encryption_context.getKeyBytes()) + end ## Disable table to ensure that the stores are not cached. command(:disable, @test_table) @@ -104,14 +134,15 @@ def setup # Confirm that the offline reading will fail with no config related to encryption Encryption.clearKeyProviderCache conf = Configuration.new($TEST_CLUSTER.getConfiguration) - conf.set(HConstants::CRYPTO_KEYPROVIDER_CONF_KEY, MockManagedKeyProvider.java_class.getName) + conf.set(HConstants::CRYPTO_MANAGED_KEYPROVIDER_CONF_KEY, + MockManagedKeyProvider.java_class.getName) # This is expected to fail with CorruptHFileException. - assert_raises(CorruptHFileException) do |e| + e = assert_raises(CorruptHFileException) do reader = HFile.createReader(fs, store_file_info.getPath, CacheConfig::DISABLED, true, conf) - assert_true(e.message.include?( - "Problem reading HFile Trailer from file #{store_file_info.getPath}" - )) end + assert_true(e.message.include?( + "Problem reading HFile Trailer from file #{store_file_info.getPath}" + )) Encryption.clearKeyProviderCache ## Enable back the table to be able to query. diff --git a/hbase-shell/src/test/ruby/shell/key_provider_keymeta_migration_test.rb b/hbase-shell/src/test/ruby/shell/key_provider_keymeta_migration_test.rb new file mode 100644 index 000000000000..978ee79e8655 --- /dev/null +++ b/hbase-shell/src/test/ruby/shell/key_provider_keymeta_migration_test.rb @@ -0,0 +1,641 @@ +# frozen_string_literal: true + +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'hbase_shell' +require 'stringio' +require 'hbase_constants' +require 'hbase/hbase' +require 'hbase/table' +require 'tempfile' +require 'fileutils' + +java_import org.apache.hadoop.conf.Configuration +java_import org.apache.hadoop.fs.FSDataInputStream +java_import org.apache.hadoop.hbase.CellUtil +java_import org.apache.hadoop.hbase.HConstants +java_import org.apache.hadoop.hbase.TableName +java_import org.apache.hadoop.hbase.client.Get +java_import org.apache.hadoop.hbase.client.Scan +java_import org.apache.hadoop.hbase.io.crypto.Encryption +java_import org.apache.hadoop.hbase.io.crypto.KeyStoreKeyProvider +java_import org.apache.hadoop.hbase.io.crypto.ManagedKeyProvider +java_import org.apache.hadoop.hbase.io.crypto.ManagedKeyStoreKeyProvider +java_import org.apache.hadoop.hbase.io.hfile.FixedFileTrailer +java_import org.apache.hadoop.hbase.io.hfile.HFile +java_import org.apache.hadoop.hbase.io.hfile.CacheConfig +java_import org.apache.hadoop.hbase.util.Bytes +java_import org.apache.hadoop.hbase.keymeta.KeymetaServiceEndpoint +java_import org.apache.hadoop.hbase.keymeta.KeymetaTableAccessor +java_import org.apache.hadoop.hbase.security.EncryptionUtil +java_import java.security.KeyStore +java_import java.security.MessageDigest +java_import javax.crypto.spec.SecretKeySpec +java_import java.io.FileOutputStream +java_import java.net.URLEncoder +java_import java.util.Base64 + +module Hbase + # Test class for key provider migration functionality + class KeyProviderKeymetaMigrationTest < Test::Unit::TestCase + include TestHelpers + + def setup + @test_timestamp = Time.now.to_i.to_s + @master_key_alias = 'masterkey' + @shared_key_alias = 'sharedkey' + @table_key_alias = 'tablelevelkey' + @cf_key1_alias = 'cfkey1' + @cf_key2_alias = 'cfkey2' + @keystore_password = 'password' + + # Test table names + @table_no_encryption = "no_enc_#{@test_timestamp}" + @table_random_key = "random_key_#{@test_timestamp}" + @table_table_key = "table_key_#{@test_timestamp}" + @table_shared_key1 = "shared1_#{@test_timestamp}" + @table_shared_key2 = "shared2_#{@test_timestamp}" + @table_cf_keys = "cf_keys_#{@test_timestamp}" + + # Unified table metadata with CFs and expected namespaces + @tables_metadata = { + @table_no_encryption => { + cfs: ['f'], + expected_namespace: { 'f' => nil }, + no_encryption: true + }, + @table_random_key => { + cfs: ['f'], + expected_namespace: { 'f' => nil } + }, + @table_table_key => { + cfs: ['f'], + expected_namespace: { 'f' => @table_table_key } + }, + @table_shared_key1 => { + cfs: ['f'], + expected_namespace: { 'f' => 'shared-global-key' } + }, + @table_shared_key2 => { + cfs: ['f'], + expected_namespace: { 'f' => 'shared-global-key' } + }, + @table_cf_keys => { + cfs: ['cf1', 'cf2'], + expected_namespace: { + 'cf1' => "#{@table_cf_keys}/cf1", + 'cf2' => "#{@table_cf_keys}/cf2" + } + } + } + + + # Setup initial KeyStoreKeyProvider + setup_old_key_provider + puts " >> Starting Cluster" + $TEST.startMiniCluster() + puts " >> Cluster started" + + setup_hbase + end + + define_test 'Test complete key provider migration' do + puts "\n=== Starting Key Provider Migration Test ===" + + # Step 1-3: Setup old provider and create tables + create_test_tables + puts "\n--- Validating initial table operations ---" + validate_pre_migration_operations(false) + + # Step 4: Setup new provider and restart + setup_new_key_provider + restart_cluster_and_validate + + # Step 5: Perform migration + migrate_tables_step_by_step + + # Step 6: Cleanup and final validation + cleanup_old_provider_and_validate + + puts "\n=== Migration Test Completed Successfully ===" + end + + private + + def setup_old_key_provider + puts "\n--- Setting up old KeyStoreKeyProvider ---" + + # Use proper test directory (similar to KeymetaTestUtils.setupTestKeyStore) + test_data_dir = $TEST_CLUSTER.getDataTestDir("old_keystore_#{@test_timestamp}").toString + FileUtils.mkdir_p(test_data_dir) + @old_keystore_file = File.join(test_data_dir, 'keystore.jceks') + puts " >> Old keystore file: #{@old_keystore_file}" + + # Create keystore with only the master key + # ENCRYPTION_KEY attributes generate their own keys and don't use keystore entries + create_keystore(@old_keystore_file, { + @master_key_alias => generate_key(@master_key_alias) + }) + + # Configure old KeyStoreKeyProvider + provider_uri = "jceks://#{File.expand_path(@old_keystore_file)}?password=#{@keystore_password}" + $TEST_CLUSTER.getConfiguration.set(HConstants::CRYPTO_KEYPROVIDER_CONF_KEY, + KeyStoreKeyProvider.java_class.name) + $TEST_CLUSTER.getConfiguration.set(HConstants::CRYPTO_KEYPROVIDER_PARAMETERS_KEY, provider_uri) + $TEST_CLUSTER.getConfiguration.set(HConstants::CRYPTO_MASTERKEY_NAME_CONF_KEY, @master_key_alias) + + puts " >> Old KeyStoreKeyProvider configured with keystore: #{@old_keystore_file}" + end + + def create_test_tables + puts "\n--- Creating test tables ---" + + # 1. Table without encryption + command(:create, @table_no_encryption, { 'NAME' => 'f' }) + puts " >> Created table #{@table_no_encryption} without encryption" + + # 2. Table with random key (no explicit key set) + command(:create, @table_random_key, { 'NAME' => 'f', 'ENCRYPTION' => 'AES' }) + puts " >> Created table #{@table_random_key} with random key" + + # 3. Table with table-level key + command(:create, @table_table_key, { 'NAME' => 'f', 'ENCRYPTION' => 'AES', + 'ENCRYPTION_KEY' => @table_key_alias }) + puts " >> Created table #{@table_table_key} with table-level key" + + # 4. First table with shared key + command(:create, @table_shared_key1, { 'NAME' => 'f', 'ENCRYPTION' => 'AES', + 'ENCRYPTION_KEY' => @shared_key_alias }) + puts " >> Created table #{@table_shared_key1} with shared key" + + # 5. Second table with shared key + command(:create, @table_shared_key2, { 'NAME' => 'f', 'ENCRYPTION' => 'AES', + 'ENCRYPTION_KEY' => @shared_key_alias }) + puts " >> Created table #{@table_shared_key2} with shared key" + + # 6. Table with column family specific keys + command(:create, @table_cf_keys, + { 'NAME' => 'cf1', 'ENCRYPTION' => 'AES', 'ENCRYPTION_KEY' => @cf_key1_alias }, + { 'NAME' => 'cf2', 'ENCRYPTION' => 'AES', 'ENCRYPTION_KEY' => @cf_key2_alias }) + puts " >> Created table #{@table_cf_keys} with CF-specific keys" + end + + def validate_pre_migration_operations(is_key_management_enabled) + @tables_metadata.each do |table_name, metadata| + puts " >> test_table_operations on table: #{table_name} with CFs: #{metadata[:cfs].join(', ')}" + if metadata[:no_encryption] + next + end + test_table_operations(table_name, metadata[:cfs]) + check_hfile_trailers_pre_migration(table_name, metadata[:cfs], is_key_management_enabled) + end + end + + def test_table_operations(table_name, column_families) + puts " >> Testing operations on table #{table_name}" + + test_table = table(table_name) + + column_families.each do |cf| + puts " >> Running put operations on CF: #{cf} in table: #{table_name}" + # Put data + test_table.put('row1', "#{cf}:col1", 'value1') + test_table.put('row2', "#{cf}:col2", 'value2') + end + + # Flush table + puts " >> Flushing table: #{table_name}" + $TEST_CLUSTER.flush(TableName.valueOf(table_name)) + + # Get data and validate + column_families.each do |cf| + puts " >> Validating data in CF: #{cf} in table: #{table_name}" + get_result = test_table.table.get(Get.new(Bytes.toBytes('row1'))) + assert_false(get_result.isEmpty) + assert_equal('value1', + Bytes.toString(get_result.getValue(Bytes.toBytes(cf), Bytes.toBytes('col1')))) + end + + puts " >> Operations validated for #{table_name}" + end + + def setup_new_key_provider + puts "\n--- Setting up new ManagedKeyStoreKeyProvider ---" + + # Use proper test directory (similar to KeymetaTestUtils.setupTestKeyStore) + test_data_dir = $TEST_CLUSTER.getDataTestDir("new_keystore_#{@test_timestamp}").toString + FileUtils.mkdir_p(test_data_dir) + @new_keystore_file = File.join(test_data_dir, 'managed_keystore.jceks') + puts " >> New keystore file: #{@new_keystore_file}" + + # Extract wrapped keys from encrypted tables and unwrap them + migrated_keys = extract_and_unwrap_keys_from_tables + + # Create new keystore with migrated keys + create_keystore(@new_keystore_file, migrated_keys) + + # Configure ManagedKeyStoreKeyProvider + provider_uri = "jceks://#{File.expand_path(@new_keystore_file)}?password=#{@keystore_password}" + $TEST_CLUSTER.getConfiguration.set(HConstants::CRYPTO_MANAGED_KEYS_ENABLED_CONF_KEY, 'true') + $TEST_CLUSTER.getConfiguration.set(HConstants::CRYPTO_MANAGED_KEYPROVIDER_CONF_KEY, + ManagedKeyStoreKeyProvider.java_class.name) + $TEST_CLUSTER.getConfiguration.set(HConstants::CRYPTO_MANAGED_KEYPROVIDER_PARAMETERS_KEY, provider_uri) + $TEST_CLUSTER.getConfiguration.set(HConstants::CRYPTO_MANAGED_KEY_STORE_SYSTEM_KEY_NAME_CONF_KEY, + 'system_key') + + # Setup key configurations for ManagedKeyStoreKeyProvider + # Shared key configuration + $TEST_CLUSTER.getConfiguration.set( + "hbase.crypto.managed_key_store.cust.#{$GLOB_CUST_ENCODED}.shared-global-key.alias", + 'shared_global_key') + $TEST_CLUSTER.getConfiguration.setBoolean( + "hbase.crypto.managed_key_store.cust.#{$GLOB_CUST_ENCODED}.shared-global-key.active", true) + + # Table-level key configuration - let system determine namespace automatically + $TEST_CLUSTER.getConfiguration.set( + "hbase.crypto.managed_key_store.cust.#{$GLOB_CUST_ENCODED}.#{@table_table_key}.alias", + "#{@table_table_key}_key") + $TEST_CLUSTER.getConfiguration.setBoolean( + "hbase.crypto.managed_key_store.cust.#{$GLOB_CUST_ENCODED}.#{@table_table_key}.active", true) + + # CF-level key configurations - let system determine namespace automatically + $TEST_CLUSTER.getConfiguration.set( + "hbase.crypto.managed_key_store.cust.#{$GLOB_CUST_ENCODED}.#{@table_cf_keys}/cf1.alias", + "#{@table_cf_keys}_cf1_key") + $TEST_CLUSTER.getConfiguration.setBoolean( + "hbase.crypto.managed_key_store.cust.#{$GLOB_CUST_ENCODED}.#{@table_cf_keys}/cf1.active", true) + + $TEST_CLUSTER.getConfiguration.set( + "hbase.crypto.managed_key_store.cust.#{$GLOB_CUST_ENCODED}.#{@table_cf_keys}/cf2.alias", + "#{@table_cf_keys}_cf2_key") + $TEST_CLUSTER.getConfiguration.setBoolean( + "hbase.crypto.managed_key_store.cust.#{$GLOB_CUST_ENCODED}.#{@table_cf_keys}/cf2.active", true) + + # Enable KeyMeta coprocessor + $TEST_CLUSTER.getConfiguration.set('hbase.coprocessor.master.classes', + KeymetaServiceEndpoint.java_class.name) + + puts " >> New ManagedKeyStoreKeyProvider configured" + end + + def restart_cluster_and_validate + puts "\n--- Restarting cluster with managed key store key provider ---" + + $TEST.restartMiniCluster(KeymetaTableAccessor::KEY_META_TABLE_NAME) + puts " >> Cluster restarted with ManagedKeyStoreKeyProvider" + setup_hbase + + # Validate key management service is functional + output = capture_stdout { command(:show_key_status, "#{$GLOB_CUST_ENCODED}:*") } + assert(output.include?('0 row(s)'), "Expected 0 rows from show_key_status, got: #{output}") + #assert(output.include?(' FAILED '), "Expected FAILED status for show_key_status, got: #{output}") + puts " >> Key management service is functional" + + # Test operations still work and check HFile trailers + puts "\n--- Validating operations after restart ---" + validate_pre_migration_operations(true) + end + + def check_hfile_trailers_pre_migration(table_name, column_families, is_key_management_enabled) + puts " >> Checking HFile trailers for #{table_name} with CFs: #{column_families.join(', ')}" + + column_families.each do |cf_name| + validate_hfile_trailer(table_name, cf_name, false, is_key_management_enabled, false) + end + end + + def migrate_tables_step_by_step + puts "\n--- Performing step-by-step table migration ---" + + # Migrate shared key tables first + migrate_shared_key_tables + + # Migrate table-level key + migrate_table_level_key + + # Migrate CF-level keys + migrate_cf_level_keys + end + + def migrate_shared_key_tables + puts "\n--- Migrating shared key tables ---" + + # Enable key management for shared global key + cust_and_namespace = "#{$GLOB_CUST_ENCODED}:shared-global-key" + output = capture_stdout { command(:enable_key_management, cust_and_namespace) } + assert(output.include?("#{$GLOB_CUST_ENCODED} shared-global-key ACTIVE"), + "Expected ACTIVE status for shared key, got: #{output}") + puts " >> Enabled key management for shared global key" + + # Migrate first shared key table + migrate_table_to_managed_key(@table_shared_key1, 'f', 'shared-global-key', true) + + # Migrate second shared key table + migrate_table_to_managed_key(@table_shared_key2, 'f', 'shared-global-key', true) + end + + def migrate_table_level_key + puts "\n--- Migrating table-level key ---" + + # Enable key management for table namespace + cust_and_namespace = "#{$GLOB_CUST_ENCODED}:#{@table_table_key}" + output = capture_stdout { command(:enable_key_management, cust_and_namespace) } + assert(output.include?("#{$GLOB_CUST_ENCODED} #{@table_table_key} ACTIVE"), + "Expected ACTIVE status for table key, got: #{output}") + puts " >> Enabled key management for table-level key" + + # Migrate the table - no namespace attribute, let system auto-determine + migrate_table_to_managed_key(@table_table_key, 'f', @table_table_key, false) + end + + def migrate_cf_level_keys + puts "\n--- Migrating CF-level keys ---" + + # Enable key management for CF1 + cf1_namespace = "#{@table_cf_keys}/cf1" + cust_and_namespace = "#{$GLOB_CUST_ENCODED}:#{cf1_namespace}" + output = capture_stdout { command(:enable_key_management, cust_and_namespace) } + assert(output.include?("#{$GLOB_CUST_ENCODED} #{cf1_namespace} ACTIVE"), + "Expected ACTIVE status for CF1 key, got: #{output}") + puts " >> Enabled key management for CF1" + + # Enable key management for CF2 + cf2_namespace = "#{@table_cf_keys}/cf2" + cust_and_namespace = "#{$GLOB_CUST_ENCODED}:#{cf2_namespace}" + output = capture_stdout { command(:enable_key_management, cust_and_namespace) } + assert(output.include?("#{$GLOB_CUST_ENCODED} #{cf2_namespace} ACTIVE"), + "Expected ACTIVE status for CF2 key, got: #{output}") + puts " >> Enabled key management for CF2" + + # Migrate CF1 + migrate_table_to_managed_key(@table_cf_keys, 'cf1', cf1_namespace, false) + + # Migrate CF2 + migrate_table_to_managed_key(@table_cf_keys, 'cf2', cf2_namespace, false) + end + + def migrate_table_to_managed_key(table_name, cf_name, namespace, use_namespace_attribute = false) + puts " >> Migrating table #{table_name}, CF #{cf_name} to namespace #{namespace}" + + # Use atomic alter operation to remove ENCRYPTION_KEY and optionally add ENCRYPTION_KEY_NAMESPACE + if use_namespace_attribute + # For shared key tables: remove ENCRYPTION_KEY and add ENCRYPTION_KEY_NAMESPACE atomically + command(:alter, table_name, + { 'NAME' => cf_name, 'CONFIGURATION' => {'ENCRYPTION_KEY' => '', 'ENCRYPTION_KEY_NAMESPACE' => namespace }}) + else + # For table/CF level keys: just remove ENCRYPTION_KEY, let system auto-determine namespace + command(:alter, table_name, + { 'NAME' => cf_name, 'CONFIGURATION' => {'ENCRYPTION_KEY' => '' }}) + end + + puts " >> Altered #{table_name} CF #{cf_name} to use namespace #{namespace}" + + # The new encryption attribute won't be used unless HStore is reinitialized. + # To force reinitialization, disable and enable the table. + command(:disable, table_name) + command(:enable, table_name) + # sleep for 5s to ensure region is reopened and store is reinitialized + sleep(5) + + # Scan all existing data to verify accessibility + scan_and_validate_table(table_name, [cf_name]) + + # Add new data + test_table = table(table_name) + test_table.put('new_row', "#{cf_name}:new_col", 'new_value') + + # Flush and validate trailer + $TEST_CLUSTER.flush(TableName.valueOf(table_name)) + validate_hfile_trailer(table_name, cf_name, true, true, false, namespace) + + puts " >> Migration completed for #{table_name} CF #{cf_name}" + end + + + def scan_and_validate_table(table_name, column_families) + puts " >> Scanning and validating existing data in #{table_name}" + + test_table = table(table_name) + scan = Scan.new + scanner = test_table.table.getScanner(scan) + + row_count = 0 + while (result = scanner.next) + row_count += 1 + assert_false(result.isEmpty) + end + scanner.close + + assert(row_count > 0, "Expected to find existing data in #{table_name}") + puts " >> Found #{row_count} rows, all accessible" + end + + def validate_hfile_trailer(table_name, cf_name, is_post_migration, is_key_management_enabled, + is_compacted, expected_namespace = nil) + context = is_post_migration ? 'migrated' : 'pre-migration' + puts " >> Validating HFile trailer for #{context} table #{table_name}, CF: #{cf_name}" + + table_name_obj = TableName.valueOf(table_name) + region_servers = $TEST_CLUSTER.getRSForFirstRegionInTable(table_name_obj) + regions = region_servers.getRegions(table_name_obj) + + regions.each do |region| + region.getStores.each do |store| + next unless store.getColumnFamilyName == cf_name + puts " >> store file count for CF: #{cf_name} in table: #{table_name} is #{store.getStorefiles.size}" + if is_compacted + assert_equal(1, store.getStorefiles.size) + else + assert_true(store.getStorefiles.size > 0) + end + store.getStorefiles.each do |storefile| + puts " >> Checking HFile trailer for storefile: #{storefile.getPath.getName} with sequence id: #{storefile.getMaxSequenceId} against max sequence id of store: #{store.getMaxSequenceId.getAsLong}" + # The flush would have created new HFiles, but the old would still be there + # so we need to make sure to check the latest store only. + next unless storefile.getMaxSequenceId == store.getMaxSequenceId.getAsLong + store_file_info = storefile.getFileInfo + next unless store_file_info + + hfile_info = store_file_info.getHFileInfo + next unless hfile_info + + trailer = hfile_info.getTrailer + + assert_not_nil(trailer.getEncryptionKey) + + if is_key_management_enabled + assert_not_nil(trailer.getKEKMetadata) + assert_not_equal(0, trailer.getKEKChecksum) + else + assert_nil(trailer.getKEKMetadata) + assert_equal(0, trailer.getKEKChecksum) + end + + if is_post_migration + assert_equal(expected_namespace, trailer.getKeyNamespace) + puts " >> Trailer validation passed - namespace: #{trailer.getKeyNamespace}" + else + assert_nil(trailer.getKeyNamespace) + puts " >> Trailer validation passed - using legacy key format" + end + end + end + end + end + + + def cleanup_old_provider_and_validate + puts "\n--- Cleaning up old key provider and final validation ---" + + # Remove old KeyProvider configurations + $TEST_CLUSTER.getConfiguration.unset(HConstants::CRYPTO_KEYPROVIDER_CONF_KEY) + $TEST_CLUSTER.getConfiguration.unset(HConstants::CRYPTO_KEYPROVIDER_PARAMETERS_KEY) + $TEST_CLUSTER.getConfiguration.unset(HConstants::CRYPTO_MASTERKEY_NAME_CONF_KEY) + + # Remove old keystore + FileUtils.rm_rf(@old_keystore_file) if File.directory?(@old_keystore_file) + puts " >> Removed old keystore and configuration" + + # Restart cluster + $TEST.restartMiniCluster(KeymetaTableAccessor::KEY_META_TABLE_NAME) + puts " >> Cluster restarted without old key provider" + setup_hbase + + # Validate all data is still accessible + validate_all_tables_final + + # Perform major compaction and validate + perform_major_compaction_and_validate + end + + def validate_all_tables_final + puts "\n--- Final validation - scanning all tables ---" + + @tables_metadata.each do |table_name, metadata| + if metadata[:no_encryption] + next + end + puts " >> Final validation for table: #{table_name} with CFs: #{metadata[:cfs].join(', ')}" + scan_and_validate_table(table_name, metadata[:cfs]) + puts " >> #{table_name} - all data accessible" + end + end + + def perform_major_compaction_and_validate + puts "\n--- Performing major compaction and final validation ---" + + $TEST_CLUSTER.compact(true) + + @tables_metadata.each do |table_name, metadata| + if metadata[:no_encryption] + next + end + puts " >> Validating post-compaction HFiles for table: #{table_name} with CFs: #{metadata[:cfs].join(', ')}" + metadata[:cfs].each do |cf_name| + # When using random key from system key, there is no namespace + #next if metadata[:expected_namespace][cf_name] == '*' + validate_hfile_trailer(table_name, cf_name, true, true, true, metadata[:expected_namespace][cf_name]) + end + end + end + + # Utility methods + + def extract_and_unwrap_keys_from_tables + puts " >> Extracting and unwrapping keys from encrypted tables" + + keys = {} + + # Reuse existing master key from old keystore as system key + old_key_provider = Encryption.getKeyProvider($TEST_CLUSTER.getConfiguration) + master_key_bytes = old_key_provider.getKey(@master_key_alias).getEncoded + keys['system_key'] = master_key_bytes + + # Extract wrapped keys from table descriptors and unwrap them + # Only call extract_key_from_table for tables that have ENCRYPTION_KEY attribute + + # For shared key tables (both use same key) + shared_key = extract_key_from_table(@table_shared_key1, 'f') + keys['shared_global_key'] = shared_key + + # For table-level key + table_key = extract_key_from_table(@table_table_key, 'f') + keys["#{@table_table_key}_key"] = table_key + + # For CF-level keys + cf1_key = extract_key_from_table(@table_cf_keys, 'cf1') + keys["#{@table_cf_keys}_cf1_key"] = cf1_key + + cf2_key = extract_key_from_table(@table_cf_keys, 'cf2') + keys["#{@table_cf_keys}_cf2_key"] = cf2_key + + puts " >> Extracted #{keys.size} keys for migration" + keys + end + + def extract_key_from_table(table_name, cf_name) + # Get table descriptor + admin = $TEST_CLUSTER.getAdmin + table_descriptor = admin.getDescriptor(TableName.valueOf(table_name)) + cf_descriptor = table_descriptor.getColumnFamily(Bytes.toBytes(cf_name)) + + # Get the wrapped key bytes from ENCRYPTION_KEY attribute + wrapped_key_bytes = cf_descriptor.getEncryptionKey + + # Use EncryptionUtil.unwrapKey with master key alias as subject + unwrapped_key = EncryptionUtil.unwrapKey($TEST_CLUSTER.getConfiguration, + @master_key_alias, wrapped_key_bytes) + + return unwrapped_key.getEncoded + end + + def generate_key(alias_name) + MessageDigest.getInstance('SHA-256').digest(Bytes.toBytes(alias_name)) + end + + def create_keystore(keystore_path, key_entries) + store = KeyStore.getInstance('JCEKS') + password_chars = @keystore_password.to_java.toCharArray + store.load(nil, password_chars) + + key_entries.each do |alias_name, key_bytes| + secret_key = SecretKeySpec.new(key_bytes, 'AES') + store.setEntry(alias_name, KeyStore::SecretKeyEntry.new(secret_key), + KeyStore::PasswordProtection.new(password_chars)) + end + + fos = FileOutputStream.new(keystore_path) + begin + store.store(fos, password_chars) + ensure + fos.close + end + end + + + def teardown + # Cleanup temporary test directories (keystore files will be cleaned up with the directories) + test_base_dir = $TEST_CLUSTER.getDataTestDir().toString + Dir.glob(File.join(test_base_dir, "*keystore_#{@test_timestamp}*")).each do |dir| + FileUtils.rm_rf(dir) if File.directory?(dir) + end + end + end +end diff --git a/hbase-shell/src/test/ruby/tests_runner.rb b/hbase-shell/src/test/ruby/tests_runner.rb index 4e31b81535a7..4c93d8d872ba 100644 --- a/hbase-shell/src/test/ruby/tests_runner.rb +++ b/hbase-shell/src/test/ruby/tests_runner.rb @@ -40,6 +40,9 @@ end files = Dir[ File.dirname(__FILE__) + "/" + test_suite_pattern ] +if files.empty? + raise "No tests found for #{test_suite_pattern}" +end files.each do |file| filename = File.basename(file) if includes != nil && !includes.include?(filename)