diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat.java index df063a4..5a3f9a7 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat.java @@ -31,6 +31,7 @@ import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.io.compress.Compression.Algorithm; +import org.apache.hadoop.hbase.io.crypto.Encryption; import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; import org.apache.hadoop.hbase.regionserver.BloomType; import org.apache.hadoop.mapreduce.Job; @@ -138,6 +139,21 @@ public class HFileOutputFormat extends FileOutputFormat createFamilyEncryptionMap( + Configuration conf) throws IOException { + return HFileOutputFormat2.createFamilyEncryptionMap(conf); + } /** * Configure job with a TotalOrderPartitioner, partitioning against @@ -206,4 +222,19 @@ public class HFileOutputFormat extends FileOutputFormat cryptoContextMap = createFamilyEncryptionMap(conf); return new RecordWriter() { // Map of families to writers and how much has been output on the writer. @@ -229,6 +242,8 @@ public class HFileOutputFormat2 DataBlockEncoding encoding = overriddenEncoding; encoding = encoding == null ? datablockEncodingMap.get(family) : encoding; encoding = encoding == null ? DataBlockEncoding.NONE : encoding; + Encryption.Context cryptoContext = cryptoContextMap.get(family); + cryptoContext = cryptoContext == null ? defaultCryptoContext : cryptoContext; Configuration tempConf = new Configuration(conf); tempConf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.0f); HFileContextBuilder contextBuilder = new HFileContextBuilder() @@ -237,6 +252,7 @@ public class HFileOutputFormat2 .withBytesPerCheckSum(HStore.getBytesPerChecksum(conf)) .withBlockSize(blockSize); contextBuilder.withDataBlockEncoding(encoding); + contextBuilder.withEncryptionContext(cryptoContext); HFileContext hFileContext = contextBuilder.build(); wl.writer = new StoreFile.WriterBuilder(conf, new CacheConfig(tempConf), fs) @@ -394,6 +410,7 @@ public class HFileOutputFormat2 configureBloomType(table, conf); configureBlockSize(table, conf); configureDataBlockEncoding(table, conf); + configureEncryption(table, conf); TableMapReduceUtil.addDependencyJars(job); TableMapReduceUtil.initCredentials(job); @@ -484,6 +501,85 @@ public class HFileOutputFormat2 return encoderMap; } + /** + * Runs inside the task to deserialize column family to encryption type + * map from the configuration. + * + * @param conf to read the serialized values from + * @return a map from column family to the configured encryption type + * @throws IOException + */ + @VisibleForTesting + static Map createFamilyEncryptionMap(Configuration conf) + throws IOException { + Map typeMap = createFamilyConfValueMap(conf, ENCRYPTION_TYPE_FAMILIES_CONF_KEY); + Map keyMap = createFamilyConfValueMap(conf, ENCRYPTION_KEY_FAMILIES_CONF_KEY); + Map encryptionMap = + new TreeMap(Bytes.BYTES_COMPARATOR); + + String masterKeyName = conf.get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, + User.getCurrent().getShortName()); + String alternateKeyName = + conf.get(HConstants.CRYPTO_MASTERKEY_ALTERNATE_NAME_CONF_KEY); + + for (Map.Entry e : typeMap.entrySet()) { + Encryption.Context cryptoContext = Encryption.Context.NONE; + String cipherName = e.getValue(); + if (cipherName != null) { + Cipher cipher; + Key key; + if (keyMap.get(e.getKey()) != null) { + byte[] keyBytes = Base64.decodeBase64(keyMap.get(e.getKey())); + // Family provides specific key material + try { + // First try the master key + key = EncryptionUtil.unwrapKey(conf, masterKeyName, keyBytes); + } catch (KeyException ke) { + // If the current master key fails to unwrap, try the alternate, if + // one is configured + if (LOG.isDebugEnabled()) { + LOG.debug("Unable to unwrap key with current master key '" + masterKeyName + "'"); + } + if (alternateKeyName != null) { + try { + key = EncryptionUtil.unwrapKey(conf, alternateKeyName, keyBytes); + } catch (KeyException ex) { + throw new IOException(ex); + } + } else { + throw new IOException(ke); + } + } + // Use the algorithm the key wants + cipher = Encryption.getCipher(conf, key.getAlgorithm()); + if (cipher == null) { + throw new IOException("Cipher '" + cipher + "' is not available"); + } + // Fail if misconfigured + // We use the encryption type specified in the column schema as a sanity check on + // what the wrapped key is telling us + if (!cipher.getName().equalsIgnoreCase(cipherName)) { + throw new IOException("Encryption for family '" + Bytes.toString(e.getKey()) + + "' configured with type '" + cipherName + + "' but key specifies algorithm '" + cipher.getName() + "'"); + } + } else { + // Family does not provide key material, create a random key + cipher = Encryption.getCipher(conf, cipherName); + if (cipher == null) { + throw new IOException("Cipher '" + cipher + "' is not available"); + } + key = cipher.getRandomKey(); + } + cryptoContext = Encryption.newContext(conf); + cryptoContext.setCipher(cipher); + cryptoContext.setKey(key); + } + + encryptionMap.put(e.getKey(), cryptoContext); + } + return encryptionMap; + } /** * Run inside the task to deserialize column family to given conf value map. @@ -674,4 +770,95 @@ public class HFileOutputFormat2 conf.set(DATABLOCK_ENCODING_FAMILIES_CONF_KEY, dataBlockEncodingConfigValue.toString()); } + + /** + * Serialize column family to encryption map to configuration. + * Invoked while configuring the MR job for incremental load. + * + * @param table to read the properties from + * @param conf to persist serialized values into + * @throws IOException + * on failure to read column family descriptors + */ + @VisibleForTesting + static void configureEncryption(HTable table, + Configuration conf) throws IOException { + configureEncryptionType(table, conf); + configureEncryptionKey(table, conf); + } + + /** + * Serialize column family to encryption type map to configuration. + * Invoked while configuring the MR job for incremental load. + * + * @param table to read the properties from + * @param conf to persist serialized values into + * @throws IOException + * on failure to read column family descriptors + */ + @VisibleForTesting + static void configureEncryptionType(HTable table, + Configuration conf) throws IOException { + HTableDescriptor tableDescriptor = table.getTableDescriptor(); + if (tableDescriptor == null) { + // could happen with mock table instance + return; + } + StringBuilder encryptionTypeConfigValue = new StringBuilder(); + Collection families = tableDescriptor.getFamilies(); + int i = 0; + for (HColumnDescriptor familyDescriptor : families) { + if (i++ > 0) { + encryptionTypeConfigValue.append('&'); + } + encryptionTypeConfigValue.append( + URLEncoder.encode(familyDescriptor.getNameAsString(), "UTF-8")); + encryptionTypeConfigValue.append('='); + String encryptionType = familyDescriptor.getEncryptionType(); + if (encryptionType == null) { + encryptionType = ""; + } + encryptionTypeConfigValue.append(URLEncoder.encode(encryptionType, + "UTF-8")); + } + conf.set(ENCRYPTION_TYPE_FAMILIES_CONF_KEY, encryptionTypeConfigValue.toString()); + } + + /** + * Serialize column family to encryption key map to configuration. + * Invoked while configuring the MR job for incremental load. + * + * @param table to read the properties from + * @param conf to persist serialized values into + * @throws IOException + * on failure to read column family descriptors + */ + @VisibleForTesting + static void configureEncryptionKey(HTable table, + Configuration conf) throws IOException { + HTableDescriptor tableDescriptor = table.getTableDescriptor(); + if (tableDescriptor == null) { + // could happen with mock table instance + return; + } + StringBuilder encryptionKeyConfigValue = new StringBuilder(); + Collection families = tableDescriptor.getFamilies(); + int i = 0; + for (HColumnDescriptor familyDescriptor : families) { + if (i++ > 0) { + encryptionKeyConfigValue.append('&'); + } + encryptionKeyConfigValue.append( + URLEncoder.encode(familyDescriptor.getNameAsString(), "UTF-8")); + encryptionKeyConfigValue.append('='); + String encryptionKey = ""; + if (familyDescriptor.getEncryptionKey() != null) { + // Use Base64 for encoding byte array + encryptionKey = new String(Base64.encodeBase64(familyDescriptor.getEncryptionKey())); + } + encryptionKeyConfigValue.append(URLEncoder.encode(encryptionKey, + "UTF-8")); + } + conf.set(ENCRYPTION_KEY_FAMILIES_CONF_KEY, encryptionKeyConfigValue.toString()); + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java index a46660e..2dd747d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java @@ -26,6 +26,8 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.IOException; +import java.security.Key; +import java.security.SecureRandom; import java.util.Arrays; import java.util.HashMap; import java.util.Map; @@ -34,6 +36,8 @@ import java.util.Random; import java.util.Set; import java.util.concurrent.Callable; +import javax.crypto.spec.SecretKeySpec; + import junit.framework.Assert; import org.apache.commons.logging.Log; @@ -64,6 +68,9 @@ import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.io.compress.Compression; import org.apache.hadoop.hbase.io.compress.Compression.Algorithm; +import org.apache.hadoop.hbase.io.crypto.Cipher; +import org.apache.hadoop.hbase.io.crypto.Encryption; +import org.apache.hadoop.hbase.io.crypto.KeyProviderForTesting; import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.io.hfile.HFile; @@ -72,6 +79,8 @@ import org.apache.hadoop.hbase.regionserver.BloomType; import org.apache.hadoop.hbase.regionserver.HStore; import org.apache.hadoop.hbase.regionserver.StoreFile; import org.apache.hadoop.hbase.regionserver.TimeRangeTracker; +import org.apache.hadoop.hbase.security.EncryptionUtil; +import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.util.Threads; @@ -767,6 +776,109 @@ public class TestHFileOutputFormat { return familyToDataBlockEncoding; } + /** + * Test for {@link HFileOutputFormat#configureEncryption(HTable, + * Configuration)} and {@link HFileOutputFormat#createFamilyEncryptionMap + * (Configuration)}. + * Tests that the encryption map is correctly serialized into + * and deserialized from configuration + * + * @throws IOException + */ + @Test + public void testSerializeDeserializeFamilyEncryptionMap() throws IOException { + for (int numCfs = 0; numCfs <= 2; numCfs++) { + Configuration conf = new Configuration(this.util.getConfiguration()); + conf.setInt("hfile.format.version", 3); + conf.set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, KeyProviderForTesting.class.getName()); + conf.set(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, "hbase"); + Map familyToEncryption = + getMockColumnFamiliesForEncryption(conf, numCfs); + HTable table = Mockito.mock(HTable.class); + setupMockColumnFamiliesForEncryption(conf, table, + familyToEncryption); + HFileOutputFormat.configureEncryption(table, conf); + + // read back family specific encryption settings from the + // configuration + Map retrievedFamilyToEncryptionMap = + HFileOutputFormat.createFamilyEncryptionMap(conf); + + // test that we have a value for all column families that matches with the + // used mock values + for (Entry entry : familyToEncryption.entrySet()) { + if (retrievedFamilyToEncryptionMap.get(entry.getKey().getBytes()) != null) { + assertEquals("Encryption configuration incorrect for column family:" + entry.getKey(), + entry.getValue().getCipher().getName(), + retrievedFamilyToEncryptionMap.get(entry.getKey().getBytes()).getCipher().getName()); + } + } + } + } + + private void setupMockColumnFamiliesForEncryption(Configuration conf, HTable table, + Map familyToEncryption) throws IOException { + HTableDescriptor mockTableDescriptor = new HTableDescriptor(TABLE_NAME); + for (Entry entry : familyToEncryption.entrySet()) { + HColumnDescriptor hcd = new HColumnDescriptor(entry.getKey()); + hcd.setMaxVersions(1); + hcd.setBlockCacheEnabled(false); + hcd.setTimeToLive(0); + if (entry.getValue().getCipher() != null) + { + hcd.setEncryptionType(entry.getValue().getCipher().getName()); + hcd.setEncryptionKey(EncryptionUtil.wrapKey(conf, + conf.get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, User.getCurrent().getShortName()), + entry.getValue().getKey())); + } + + mockTableDescriptor.addFamily(hcd); + } + Mockito.doReturn(mockTableDescriptor).when(table).getTableDescriptor(); + } + + /** + * @return a map from column family names to encryption type for + * testing column family Encryption. Column family names have special characters + */ + private Map + getMockColumnFamiliesForEncryption (Configuration conf, int numCfs) { + Map familyToEncryption = + new HashMap(); + // use column family names having special characters + if (numCfs-- >= 0) { + familyToEncryption.put("Family1!@#!@#&", Encryption.Context.NONE); + } + if (numCfs-- >= 0) { + Encryption.Context cryptoContext = Encryption.newContext(conf); + Cipher aes = Encryption.getCipher(conf, "AES"); + assertNotNull(aes); + cryptoContext.setCipher(aes); + byte[] keyBytes = new byte[aes.getKeyLength()]; + SecureRandom RNG = new SecureRandom(); + RNG.nextBytes(keyBytes); + Key key = new SecretKeySpec(keyBytes, aes.getName()); + cryptoContext.setKey(key); + familyToEncryption.put("Family2=asdads&!AASD", + cryptoContext); + } + + if (numCfs-- >= 0) { + Encryption.Context cryptoContext = Encryption.newContext(conf); + Cipher aes = Encryption.getCipher(conf, "AES"); + assertNotNull(aes); + cryptoContext.setCipher(aes); + byte[] keyBytes = new byte[aes.getKeyLength()]; + SecureRandom RNG = new SecureRandom(); + RNG.nextBytes(keyBytes); + Key key = new SecretKeySpec(keyBytes, aes.getName()); + cryptoContext.setKey(key); + familyToEncryption.put("Family3", + cryptoContext); + } + return familyToEncryption; + } + private void setupMockStartKeys(HTable table) throws IOException { byte[][] mockKeys = new byte[][] { HConstants.EMPTY_BYTE_ARRAY, diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java index 76db299..2397260 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java @@ -26,6 +26,8 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.IOException; +import java.security.Key; +import java.security.SecureRandom; import java.util.Arrays; import java.util.HashMap; import java.util.Map; @@ -34,6 +36,8 @@ import java.util.Random; import java.util.Set; import java.util.concurrent.Callable; +import javax.crypto.spec.SecretKeySpec; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -62,6 +66,9 @@ import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.io.compress.Compression; import org.apache.hadoop.hbase.io.compress.Compression.Algorithm; +import org.apache.hadoop.hbase.io.crypto.Cipher; +import org.apache.hadoop.hbase.io.crypto.Encryption; +import org.apache.hadoop.hbase.io.crypto.KeyProviderForTesting; import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.io.hfile.HFile; @@ -69,6 +76,8 @@ import org.apache.hadoop.hbase.io.hfile.HFile.Reader; import org.apache.hadoop.hbase.regionserver.BloomType; import org.apache.hadoop.hbase.regionserver.StoreFile; import org.apache.hadoop.hbase.regionserver.TimeRangeTracker; +import org.apache.hadoop.hbase.security.EncryptionUtil; +import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.util.Threads; @@ -778,6 +787,103 @@ public class TestHFileOutputFormat2 { } /** + * Test for {@link HFileOutputFormat2#configureEncryption(HTable, + * Configuration)} and {@link HFileOutputFormat2#createFamilyEncryptionMap + * (Configuration)}. + * Tests that the encryption map is correctly serialized into + * and deserialized from configuration + * + * @throws IOException + */ + @Test + public void testSerializeDeserializeFamilyEncryptionMap() throws IOException { + for (int numCfs = 0; numCfs <= 2; numCfs++) { + Configuration conf = new Configuration(this.util.getConfiguration()); + conf.setInt("hfile.format.version", 3); + conf.set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, KeyProviderForTesting.class.getName()); + conf.set(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, "hbase"); + Map familyToEncryption = + getMockColumnFamiliesForEncryption(conf, numCfs); + HTable table = Mockito.mock(HTable.class); + setupMockColumnFamiliesForEncryption(conf, table, familyToEncryption); + HFileOutputFormat2.configureEncryption(table, conf); + + // read back family specific encryption settings from the + // configuration + Map retrievedFamilyToEncryptionMap = + HFileOutputFormat2.createFamilyEncryptionMap(conf); + + // test that we have a value for all column families that matches with the + // used mock values + for (Entry entry : familyToEncryption.entrySet()) { + if (retrievedFamilyToEncryptionMap.get(entry.getKey().getBytes()) != null) { + assertEquals("Encryption configuration incorrect for column family:" + entry.getKey(), + entry.getValue().getCipher().getName(), + retrievedFamilyToEncryptionMap.get(entry.getKey().getBytes()).getCipher().getName()); + } + } + } + } + + private void setupMockColumnFamiliesForEncryption(Configuration conf, HTable table, + Map familyToEncryption) throws IOException { + HTableDescriptor mockTableDescriptor = new HTableDescriptor(TABLE_NAME); + for (Entry entry : familyToEncryption.entrySet()) { + HColumnDescriptor hcd = new HColumnDescriptor(entry.getKey()); + hcd.setMaxVersions(1); + hcd.setBlockCacheEnabled(false); + hcd.setTimeToLive(0); + if (entry.getValue().getCipher() != null) { + hcd.setEncryptionType(entry.getValue().getCipher().getName()); + hcd.setEncryptionKey(EncryptionUtil.wrapKey(conf, + conf.get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, User.getCurrent().getShortName()), + entry.getValue().getKey())); + } + + mockTableDescriptor.addFamily(hcd); + } + Mockito.doReturn(mockTableDescriptor).when(table).getTableDescriptor(); + } + + /** + * @return a map from column family names to encryption type for + * testing column family Encryption. Column family names have special characters + */ + private Map getMockColumnFamiliesForEncryption(Configuration conf, + int numCfs) { + Map familyToEncryption = new HashMap(); + if (numCfs-- >= 0) { + familyToEncryption.put("Family1!@#!@#&", Encryption.Context.NONE); + } + if (numCfs-- >= 0) { + Encryption.Context cryptoContext = Encryption.newContext(conf); + Cipher aes = Encryption.getCipher(conf, "AES"); + assertNotNull(aes); + cryptoContext.setCipher(aes); + byte[] keyBytes = new byte[aes.getKeyLength()]; + SecureRandom RNG = new SecureRandom(); + RNG.nextBytes(keyBytes); + Key key = new SecretKeySpec(keyBytes, aes.getName()); + cryptoContext.setKey(key); + familyToEncryption.put("Family2=asdads&!AASD", cryptoContext); + } + + if (numCfs-- >= 0) { + Encryption.Context cryptoContext = Encryption.newContext(conf); + Cipher aes = Encryption.getCipher(conf, "AES"); + assertNotNull(aes); + cryptoContext.setCipher(aes); + byte[] keyBytes = new byte[aes.getKeyLength()]; + SecureRandom RNG = new SecureRandom(); + RNG.nextBytes(keyBytes); + Key key = new SecretKeySpec(keyBytes, aes.getName()); + cryptoContext.setKey(key); + familyToEncryption.put("Family3", cryptoContext); + } + return familyToEncryption; + } + + /** * Test that {@link HFileOutputFormat2} RecordWriter uses compression and * bloom filter settings from the column family descriptor */ -- 1.9.2.msysgit.0