From cb1c7b2568406a0b4162f4d40c26dd31009cfd9d Mon Sep 17 00:00:00 2001 From: Ashish Singhi Date: Thu, 12 Feb 2015 20:19:33 +0530 Subject: [PATCH] HBASE-13002 Make encryption cipher configurable --- .../hadoop/hbase/security/EncryptionUtil.java | 20 +++++++------- .../hadoop/hbase/security/TestEncryptionUtil.java | 4 ++- .../java/org/apache/hadoop/hbase/HConstants.java | 18 ++++++++++--- .../apache/hadoop/hbase/io/crypto/Encryption.java | 31 +++++++++++++++++++--- .../hadoop/hbase/io/crypto/TestCipherProvider.java | 8 +++--- .../hadoop/hbase/io/crypto/TestEncryption.java | 7 +++-- .../apache/hadoop/hbase/regionserver/HStore.java | 4 +-- .../regionserver/wal/SecureProtobufLogWriter.java | 5 ++-- .../hadoop/hbase/io/hfile/TestHFileEncryption.java | 4 ++- .../regionserver/TestEncryptionKeyRotation.java | 14 +++++++--- .../regionserver/TestEncryptionRandomKeying.java | 4 ++- .../hadoop/hbase/util/TestEncryptionTest.java | 6 +++-- .../hadoop/hbase/util/TestHBaseFsckEncryption.java | 6 +++-- 13 files changed, 93 insertions(+), 38 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/EncryptionUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/EncryptionUtil.java index f4bc3e9..e877363 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/EncryptionUtil.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/EncryptionUtil.java @@ -63,8 +63,7 @@ public class EncryptionUtil { /** * Protect a key by encrypting it with the secret key of the given subject. - * The configuration must be set up correctly for key alias resolution. Keys - * are always wrapped using AES. + * The configuration must be set up correctly for key alias resolution. * @param conf configuration * @param subject subject key alias * @param key the key @@ -72,10 +71,12 @@ public class EncryptionUtil { */ public static byte[] wrapKey(Configuration conf, String subject, Key key) throws IOException { - // Wrap the key with AES - Cipher cipher = Encryption.getCipher(conf, "AES"); + // Wrap the key with the configured encryption algorithm. + String algorithm = + conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES); + Cipher cipher = Encryption.getCipher(conf, algorithm); if (cipher == null) { - throw new RuntimeException("Cipher 'AES' not available"); + throw new RuntimeException("Cipher '" + algorithm + "' not available"); } EncryptionProtos.WrappedKey.Builder builder = EncryptionProtos.WrappedKey.newBuilder(); builder.setAlgorithm(key.getAlgorithm()); @@ -100,8 +101,7 @@ public class EncryptionUtil { /** * Unwrap a key by decrypting it with the secret key of the given subject. - * The configuration must be set up correctly for key alias resolution. Keys - * are always unwrapped using AES. + * The configuration must be set up correctly for key alias resolution. * @param conf configuration * @param subject subject key alias * @param value the encrypted key bytes @@ -113,9 +113,11 @@ public class EncryptionUtil { throws IOException, KeyException { EncryptionProtos.WrappedKey wrappedKey = EncryptionProtos.WrappedKey.PARSER .parseDelimitedFrom(new ByteArrayInputStream(value)); - Cipher cipher = Encryption.getCipher(conf, "AES"); + String algorithm = conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, + HConstants.CIPHER_AES); + Cipher cipher = Encryption.getCipher(conf, algorithm); if (cipher == null) { - throw new RuntimeException("Algorithm 'AES' not available"); + throw new RuntimeException("Cipher '" + algorithm + "' not available"); } ByteArrayOutputStream out = new ByteArrayOutputStream(); byte[] iv = wrappedKey.hasIv() ? wrappedKey.getIv().toByteArray() : null; diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/security/TestEncryptionUtil.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/security/TestEncryptionUtil.java index ed6f49b..f9dd30b 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/security/TestEncryptionUtil.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/security/TestEncryptionUtil.java @@ -49,7 +49,9 @@ public class TestEncryptionUtil { // generate a test key byte[] keyBytes = new byte[AES.KEY_LENGTH]; new SecureRandom().nextBytes(keyBytes); - Key key = new SecretKeySpec(keyBytes, "AES"); + String algorithm = + conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES); + Key key = new SecretKeySpec(keyBytes, algorithm); // wrap the test key byte[] wrappedKeyBytes = EncryptionUtil.wrapKey(conf, "hbase", key); diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java index 2ee55f7..42773e6 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java @@ -388,7 +388,7 @@ public final class HConstants { /** * The hbase:meta table's name. - * + * */ @Deprecated // for compat from 0.94 -> 0.96. public static final byte[] META_TABLE_NAME = TableName.META_TABLE_NAME.getName(); @@ -929,7 +929,7 @@ public final class HConstants { * NONE: no preference in destination of replicas * ONE_SSD: place only one replica in SSD and the remaining in default storage * and ALL_SSD: place all replica on SSD - * + * * See http://hadoop.apache.org/docs/r2.6.0/hadoop-project-dist/hadoop-hdfs/ArchivalStorage.html*/ public static final String WAL_STORAGE_POLICY = "hbase.wal.storage.policy"; public static final String DEFAULT_WAL_STORAGE_POLICY = "NONE"; @@ -1041,6 +1041,9 @@ public final class HConstants { public static final long NO_NONCE = 0; + /** Default cipher for encryption */ + public static final String CIPHER_AES = "AES"; + /** Configuration key for the crypto algorithm provider, a class name */ public static final String CRYPTO_CIPHERPROVIDER_CONF_KEY = "hbase.crypto.cipherprovider"; @@ -1064,6 +1067,13 @@ public final class HConstants { /** Configuration key for the name of the master WAL encryption key for the cluster, a string */ public static final String CRYPTO_WAL_KEY_NAME_CONF_KEY = "hbase.crypto.wal.key.name"; + /** Configuration key for the algorithm used for creating jks key, a string */ + public static final String CRYPTO_KEY_ALGORITHM_CONF_KEY = "hbase.crypto.key.algorithm"; + + /** Configuration key for the name of the alternate cipher algorithm for the cluster, a string */ + public static final String CRYPTO_ALTERNATE_KEY_ALGORITHM_CONF_KEY = + "hbase.crypto.alternate.key.algorithm"; + /** Configuration key for enabling WAL encryption, a boolean */ public static final String ENABLE_WAL_ENCRYPTION = "hbase.regionserver.wal.encryption"; @@ -1116,7 +1126,7 @@ public final class HConstants { public static final String HBASE_CLIENT_FAST_FAIL_THREASHOLD_MS = "hbase.client.fastfail.threshold"; - + public static final long HBASE_CLIENT_FAST_FAIL_THREASHOLD_MS_DEFAULT = 60000; @@ -1127,7 +1137,7 @@ public final class HConstants { 600000; public static final String HBASE_CLIENT_FAST_FAIL_INTERCEPTOR_IMPL = - "hbase.client.fast.fail.interceptor.impl"; + "hbase.client.fast.fail.interceptor.impl"; /** Config key for if the server should send backpressure and if the client should listen to * that backpressure from the server */ diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Encryption.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Encryption.java index 3420d0a..ad89ca0 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Encryption.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Encryption.java @@ -469,9 +469,8 @@ public final class Encryption { * @param iv the initialization vector, can be null * @throws IOException */ - public static void decryptWithSubjectKey(OutputStream out, InputStream in, - int outLen, String subject, Configuration conf, Cipher cipher, - byte[] iv) throws IOException { + public static void decryptWithSubjectKey(OutputStream out, InputStream in, int outLen, + String subject, Configuration conf, Cipher cipher, byte[] iv) throws IOException { Key key = getSecretKeyForSubject(subject, conf); if (key == null) { throw new IOException("No key found for subject '" + subject + "'"); @@ -479,7 +478,31 @@ public final class Encryption { Decryptor d = cipher.getDecryptor(); d.setKey(key); d.setIv(iv); // can be null - decrypt(out, in, outLen, d); + try { + decrypt(out, in, outLen, d); + } catch (IOException e) { + // If the current cipher algorithm fails to unwrap, try the alternate cipher algorithm, if one + // is configured + String alternateAlgorithm = conf.get(HConstants.CRYPTO_ALTERNATE_KEY_ALGORITHM_CONF_KEY); + if (alternateAlgorithm != null) { + if (LOG.isDebugEnabled()) { + LOG.debug("Unable to decrypt data with current cipher algorithm '" + + conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES) + + "'. Trying with the alternate cipher algorithm '" + alternateAlgorithm + + "' configured."); + } + Cipher alterCipher = Encryption.getCipher(conf, alternateAlgorithm); + if (alterCipher == null) { + throw new RuntimeException("Cipher '" + alternateAlgorithm + "' not available"); + } + d = alterCipher.getDecryptor(); + d.setKey(key); + d.setIv(iv); // can be null + decrypt(out, in, outLen, d); + } else { + throw new IOException(e); + } + } } private static ClassLoader getClassLoaderForClass(Class c) { diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/TestCipherProvider.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/TestCipherProvider.java index fdb9448..dbf7fc5 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/TestCipherProvider.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/TestCipherProvider.java @@ -142,11 +142,13 @@ public class TestCipherProvider { Configuration conf = HBaseConfiguration.create(); CipherProvider provider = Encryption.getCipherProvider(conf); assertTrue(provider instanceof DefaultCipherProvider); - assertTrue(Arrays.asList(provider.getSupportedCiphers()).contains("AES")); - Cipher a = Encryption.getCipher(conf, "AES"); + String algorithm = + conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES); + assertTrue(Arrays.asList(provider.getSupportedCiphers()).contains(algorithm)); + Cipher a = Encryption.getCipher(conf, algorithm); assertNotNull(a); assertTrue(a.getProvider() instanceof DefaultCipherProvider); - assertEquals(a.getName(), "AES"); + assertEquals(a.getName(), algorithm); assertEquals(a.getKeyLength(), AES.KEY_LENGTH); } diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/TestEncryption.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/TestEncryption.java index d36333e..0d38356 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/TestEncryption.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/TestEncryption.java @@ -29,6 +29,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; @@ -89,8 +90,10 @@ public class TestEncryption { LOG.info("checkTransformSymmetry: AES, plaintext length = " + plaintext.length); Configuration conf = HBaseConfiguration.create(); - Cipher aes = Encryption.getCipher(conf, "AES"); - Key key = new SecretKeySpec(keyBytes, "AES"); + String algorithm = + conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES); + Cipher aes = Encryption.getCipher(conf, algorithm); + Key key = new SecretKeySpec(keyBytes, algorithm); Encryptor e = aes.getEncryptor(); e.setKey(key); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java index 252e5e1..b4c374d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java @@ -318,7 +318,7 @@ public class HStore implements Store { // Use the algorithm the key wants cipher = Encryption.getCipher(conf, key.getAlgorithm()); if (cipher == null) { - throw new RuntimeException("Cipher '" + cipher + "' is not available"); + throw new RuntimeException("Cipher '" + key.getAlgorithm() + "' is not available"); } // Fail if misconfigured // We use the encryption type specified in the column schema as a sanity check on @@ -332,7 +332,7 @@ public class HStore implements Store { // Family does not provide key material, create a random key cipher = Encryption.getCipher(conf, cipherName); if (cipher == null) { - throw new RuntimeException("Cipher '" + cipher + "' is not available"); + throw new RuntimeException("Cipher '" + cipherName + "' is not available"); } key = cipher.getRandomKey(); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SecureProtobufLogWriter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SecureProtobufLogWriter.java index e850485..c352770 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SecureProtobufLogWriter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SecureProtobufLogWriter.java @@ -43,8 +43,6 @@ import org.apache.hadoop.hbase.security.User; public class SecureProtobufLogWriter extends ProtobufLogWriter { private static final Log LOG = LogFactory.getLog(SecureProtobufLogWriter.class); - private static final String DEFAULT_CIPHER = "AES"; - private Encryptor encryptor = null; @Override @@ -56,7 +54,8 @@ public class SecureProtobufLogWriter extends ProtobufLogWriter { EncryptionTest.testCipherProvider(conf); // Get an instance of our cipher - final String cipherName = conf.get(HConstants.CRYPTO_WAL_ALGORITHM_CONF_KEY, DEFAULT_CIPHER); + final String cipherName = + conf.get(HConstants.CRYPTO_WAL_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES); Cipher cipher = Encryption.getCipher(conf, cipherName); if (cipher == null) { throw new RuntimeException("Cipher '" + cipherName + "' is not available"); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileEncryption.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileEncryption.java index 0cb3c3c..2d821ae 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileEncryption.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileEncryption.java @@ -70,7 +70,9 @@ public class TestHFileEncryption { fs = FileSystem.get(conf); cryptoContext = Encryption.newContext(conf); - Cipher aes = Encryption.getCipher(conf, "AES"); + String algorithm = + conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES); + Cipher aes = Encryption.getCipher(conf, algorithm); assertNotNull(aes); cryptoContext.setCipher(aes); byte[] key = new byte[aes.getKeyLength()]; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestEncryptionKeyRotation.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestEncryptionKeyRotation.java index af86b4e..b791fdb 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestEncryptionKeyRotation.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestEncryptionKeyRotation.java @@ -67,9 +67,11 @@ public class TestEncryptionKeyRotation { SecureRandom rng = new SecureRandom(); byte[] keyBytes = new byte[AES.KEY_LENGTH]; rng.nextBytes(keyBytes); - initialCFKey = new SecretKeySpec(keyBytes, "AES"); + String algorithm = + conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES); + initialCFKey = new SecretKeySpec(keyBytes, algorithm); rng.nextBytes(keyBytes); - secondCFKey = new SecretKeySpec(keyBytes, "AES"); + secondCFKey = new SecretKeySpec(keyBytes, algorithm); } @BeforeClass @@ -95,7 +97,9 @@ public class TestEncryptionKeyRotation { HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("default", "testCFKeyRotation")); HColumnDescriptor hcd = new HColumnDescriptor("cf"); - hcd.setEncryptionType("AES"); + String algorithm = + conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES); + hcd.setEncryptionType(algorithm); hcd.setEncryptionKey(EncryptionUtil.wrapKey(conf, "hbase", initialCFKey)); htd.addFamily(hcd); @@ -154,7 +158,9 @@ public class TestEncryptionKeyRotation { HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("default", "testMasterKeyRotation")); HColumnDescriptor hcd = new HColumnDescriptor("cf"); - hcd.setEncryptionType("AES"); + String algorithm = + conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES); + hcd.setEncryptionType(algorithm); hcd.setEncryptionKey(EncryptionUtil.wrapKey(conf, "hbase", initialCFKey)); htd.addFamily(hcd); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestEncryptionRandomKeying.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestEncryptionRandomKeying.java index 29a58a6..ebfc89c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestEncryptionRandomKeying.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestEncryptionRandomKeying.java @@ -92,7 +92,9 @@ public class TestEncryptionRandomKeying { // Specify an encryption algorithm without a key htd = new HTableDescriptor(TableName.valueOf("default", "TestEncryptionRandomKeying")); HColumnDescriptor hcd = new HColumnDescriptor("cf"); - hcd.setEncryptionType("AES"); + String algorithm = + conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES); + hcd.setEncryptionType(algorithm); htd.addFamily(hcd); // Start the minicluster diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestEncryptionTest.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestEncryptionTest.java index d615a29..5d2f04f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestEncryptionTest.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestEncryptionTest.java @@ -75,10 +75,12 @@ public class TestEncryptionTest { public void testTestCipher() { Configuration conf = HBaseConfiguration.create(); conf.set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, KeyProviderForTesting.class.getName()); + String algorithm = + conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES); try { - EncryptionTest.testEncryption(conf, "AES", null); + EncryptionTest.testEncryption(conf, algorithm, null); } catch (Exception e) { - fail("Test for cipher AES should have succeeded"); + fail("Test for cipher " + algorithm + " should have succeeded"); } try { EncryptionTest.testEncryption(conf, "foobar", null); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckEncryption.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckEncryption.java index 69ffa55..7c289a1 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckEncryption.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckEncryption.java @@ -78,7 +78,9 @@ public class TestHBaseFsckEncryption { SecureRandom rng = new SecureRandom(); byte[] keyBytes = new byte[AES.KEY_LENGTH]; rng.nextBytes(keyBytes); - cfKey = new SecretKeySpec(keyBytes, "AES"); + String algorithm = + conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES); + cfKey = new SecretKeySpec(keyBytes,algorithm); // Start the minicluster TEST_UTIL.startMiniCluster(3); @@ -86,7 +88,7 @@ public class TestHBaseFsckEncryption { // Create the table htd = new HTableDescriptor(TableName.valueOf("default", "TestHBaseFsckEncryption")); HColumnDescriptor hcd = new HColumnDescriptor("cf"); - hcd.setEncryptionType("AES"); + hcd.setEncryptionType(algorithm); hcd.setEncryptionKey(EncryptionUtil.wrapKey(conf, conf.get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, User.getCurrent().getShortName()), cfKey)); -- 1.9.2.msysgit.0