diff --git .gitignore .gitignore index c5decaf..099e0c9 100644 --- .gitignore +++ .gitignore @@ -27,3 +27,5 @@ hcatalog/webhcat/java-client/target hcatalog/storage-handlers/hbase/target hcatalog/webhcat/svr/target conf/hive-default.xml.template +data/files/test.jks +data/files/.test.jks.crc diff --git data/scripts/q_test_cleanup_for_encryption.sql data/scripts/q_test_cleanup_for_encryption.sql new file mode 100644 index 0000000..d28f406 --- /dev/null +++ data/scripts/q_test_cleanup_for_encryption.sql @@ -0,0 +1,5 @@ +DROP DATABASE encryptedWith128BitsKeyDB; + +DROP DATABASE encryptedWith256BitsKeyDB; + +DROP DATABASE unencryptedDB; \ No newline at end of file diff --git data/scripts/q_test_init_for_encryption.sql data/scripts/q_test_init_for_encryption.sql new file mode 100644 index 0000000..9245508 --- /dev/null +++ data/scripts/q_test_init_for_encryption.sql @@ -0,0 +1,5 @@ +CREATE DATABASE encryptedWith128BitsKeyDB; + +CREATE DATABASE encryptedWith256BitsKeyDB; + +CREATE DATABASE unencryptedDB; \ No newline at end of file diff --git itests/qtest/pom.xml itests/qtest/pom.xml index 376f4a9..43c950f 100644 --- itests/qtest/pom.xml +++ itests/qtest/pom.xml @@ -532,6 +532,24 @@ hadoopVersion="${active.hadoop.version}" initScript="q_test_init.sql" cleanupScript="q_test_cleanup.sql"/> + + diff --git itests/src/test/resources/testconfiguration.properties itests/src/test/resources/testconfiguration.properties index 3ae001d..5bf24b5 100644 --- itests/src/test/resources/testconfiguration.properties +++ itests/src/test/resources/testconfiguration.properties @@ -271,6 +271,8 @@ minitez.query.files=bucket_map_join_tez1.q,\ tez_smb_1.q,\ vectorized_dynamic_partition_pruning.q +encrypted.query.files.shared= + beeline.positive.exclude=add_part_exist.q,\ alter1.q,\ alter2.q,\ diff --git itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java index 31d5c29..9d481b9 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java +++ itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java @@ -39,6 +39,7 @@ import java.io.Serializable; import java.io.StringWriter; import java.net.URL; +import java.security.NoSuchAlgorithmException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -76,6 +77,7 @@ import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.lockmgr.zookeeper.ZooKeeperHiveLockManager; import org.apache.hadoop.hive.ql.metadata.Hive; +import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.parse.ASTNode; import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer; @@ -102,6 +104,20 @@ public class QTestUtil { public static final String UTF_8 = "UTF-8"; + + // database names used for testing the encrypted databases + public static final String encryptedWith128BitsKeyDBName = "encryptedwith128bitskeydb"; + public static final String encryptedWith256BitsKeyDBName = "encryptedwith256bitskeydb"; + public static final String unencryptedDBName = "unencrypteddb"; + + // security property names + public static final String securityKeyBitLengthPropName = "hadoop.security.key.default.bitlength"; + public static final String securityKeyCipherName = "hadoop.security.key.default.cipher"; + + // keyNames used for encrypting the hdfs path + private final String keyNameIn128 = "k128"; + private final String keyNameIn256 = "k256"; + private static final Log LOG = LogFactory.getLog("QTestUtil"); private static final String QTEST_LEAVE_FILES = "QTEST_LEAVE_FILES"; private final String defaultInitScript = "q_test_init.sql"; @@ -130,6 +146,7 @@ private CliDriver cliDriver; private HadoopShims.MiniMrShim mr = null; private HadoopShims.MiniDFSShim dfs = null; + private HadoopShims.HdfsEncryptionShim hes = null; private boolean miniMr = false; private String hadoopVer = null; private QTestSetup setup = null; @@ -245,6 +262,13 @@ private String getHadoopMainVersion(String input) { return null; } + public void initEncryptionRelatedConf() { + HadoopShims shims = ShimLoader.getHadoopShims(); + // set up the java key provider for encrypted hdfs cluster + conf.set(shims.getHadoopConfNames().get("HADOOPSECURITYKEYPROVIDER"), getKeyProviderURI()); + conf.set(securityKeyCipherName, "AES/CTR/NoPadding"); + } + public void initConf() throws Exception { String vectorizationEnabled = System.getProperty("test.vectorization.enabled"); @@ -280,6 +304,7 @@ public void initConf() throws Exception { public enum MiniClusterType { mr, tez, + encrypted, none; public static MiniClusterType valueForString(String type) { @@ -287,6 +312,8 @@ public static MiniClusterType valueForString(String type) { return mr; } else if (type.equals("tez")) { return tez; + } else if (type.equals("encrypted")) { + return encrypted; } else { return none; } @@ -299,6 +326,17 @@ public QTestUtil(String outDir, String logDir, MiniClusterType clusterType, Stri this(outDir, logDir, clusterType, null, hadoopVer, initScript, cleanupScript); } + private String getKeyProviderURI() { + // Use the current directory if it is not specified + String dataDir = conf.get("test.data.files"); + if (dataDir == null) { + dataDir = new File(".").getAbsolutePath() + "/data/files"; + } + + // put the jks file in the current test path only for test purpose + return "jceks://file" + new Path(dataDir, "test.jks").toUri(); + } + public QTestUtil(String outDir, String logDir, MiniClusterType clusterType, String confDir, String hadoopVer, String initScript, String cleanupScript) throws Exception { @@ -323,8 +361,21 @@ public QTestUtil(String outDir, String logDir, MiniClusterType clusterType, int numberOfDataNodes = 4; if (clusterType != MiniClusterType.none) { - dfs = shims.getMiniDfs(conf, numberOfDataNodes, true, null); - FileSystem fs = dfs.getFileSystem(); + FileSystem fs; + + if (clusterType == MiniClusterType.encrypted) { + initEncryptionRelatedConf(); + + dfs = shims.getMiniDfs(conf, numberOfDataNodes, true, null); + fs = dfs.getFileSystem(); + // set up the java key provider for encrypted hdfs cluster + hes = shims.createHdfsEncryptionShim(fs, conf); + System.out.println("key provider is initialized"); + } else { + dfs = shims.getMiniDfs(conf, numberOfDataNodes, true, null); + fs = dfs.getFileSystem(); + } + String uriString = WindowsPathUtil.getHdfsUriString(fs.getUri().toString()); if (clusterType == MiniClusterType.tez) { mr = shims.getMiniTezCluster(conf, 4, uriString, 1); @@ -340,7 +391,6 @@ public QTestUtil(String outDir, String logDir, MiniClusterType clusterType, if (dataDir == null) { dataDir = new File(".").getAbsolutePath() + "/data/files"; } - testFiles = dataDir; // Use the current directory if it is not specified @@ -368,7 +418,7 @@ public void shutdown() throws Exception { if (System.getenv(QTEST_LEAVE_FILES) == null) { cleanUp(); } - + setup.tearDown(); if (mr != null) { mr.shutdown(); @@ -538,6 +588,19 @@ public void clearPostTestEffects() throws Exception { } /** + * For the security type, we should reserve the encrypted databases for the test purpose + */ + private boolean checkDBIfNeedToBeReserved(String dbName) { + if (clusterType == MiniClusterType.encrypted) { + return (DEFAULT_DATABASE_NAME.equals(dbName) || + encryptedWith128BitsKeyDBName.equals(dbName) || + encryptedWith256BitsKeyDBName.equals(dbName) || unencryptedDBName.equals(dbName)); + } else { + return DEFAULT_DATABASE_NAME.equals(dbName); + } + } + + /** * Clear out any side effects of running tests */ public void clearTestSideEffects() throws Exception { @@ -545,11 +608,11 @@ public void clearTestSideEffects() throws Exception { return; } // Delete any tables other than the source tables - // and any databases other than the default database. + // and any databases other than the default database or encrypted dbs in encryption mode. for (String dbName : db.getAllDatabases()) { SessionState.get().setCurrentDatabase(dbName); for (String tblName : db.getAllTables()) { - if (!DEFAULT_DATABASE_NAME.equals(dbName) || !srcTables.contains(tblName)) { + if (!checkDBIfNeedToBeReserved(dbName) || !srcTables.contains(tblName)) { Table tblObj = db.getTable(tblName); // dropping index table can not be dropped directly. Dropping the base // table will automatically drop all its index table @@ -567,7 +630,7 @@ public void clearTestSideEffects() throws Exception { } } } - if (!DEFAULT_DATABASE_NAME.equals(dbName)) { + if (!checkDBIfNeedToBeReserved(dbName)) { // Drop cascade, may need to drop functions db.dropDatabase(dbName, true, true, true); } @@ -593,11 +656,15 @@ public void clearTestSideEffects() throws Exception { db.dropRole(roleName); } } - // allocate and initialize a new conf since a test can - // modify conf by using 'set' commands - conf = new HiveConf (Driver.class); - initConf(); - db = Hive.get(conf); // propagate new conf to meta store + + if (clusterType != MiniClusterType.encrypted) { + // allocate and initialize a new conf since a test can + // modify conf by using 'set' commands + conf = new HiveConf (Driver.class); + initConf(); + // renew the metastore since the cluster type is unencrypted + db = Hive.get(conf); // propagate new conf to meta store + } setup.preTest(conf); } @@ -685,6 +752,10 @@ public void createSources() throws Exception { cliDriver.processLine(initCommands); conf.setBoolean("hive.test.init.phase", false); + + if (clusterType == MiniClusterType.encrypted) { + initEncryptionZone(); + } } public void init() throws Exception { @@ -705,6 +776,28 @@ public void init() throws Exception { sem = new SemanticAnalyzer(conf); } + private void initEncryptionZone() throws IOException, NoSuchAlgorithmException, HiveException { + // current only aes/ctr/nopadding cipher is supported + String cipher = "AES/CTR/NoPadding"; + conf.set(securityKeyCipherName, cipher); + + // create encryption zone via a 128-bits key respectively for encrypted database 1 + conf.set(securityKeyBitLengthPropName, "128"); + + hes.createKey(keyNameIn128, conf); + hes.createEncryptionZone(new Path(db.getDatabase("encryptedWith128BitsKeyDB").getLocationUri()), + keyNameIn128); + + // create encryption zone via a 256-bits key respectively for encrypted database 2 + conf.set(securityKeyBitLengthPropName, "256"); + + // AES-256 can be used only if JCE is installed in your environment. Otherwise, any encryption + // with this key will fail. Keys can be created, but when you try to encrypt something, fails. + hes.createKey(keyNameIn256, conf); + hes.createEncryptionZone(new Path(db.getDatabase("encryptedWith256BitsKeyDB").getLocationUri()), + keyNameIn256); + } + public void init(String tname) throws Exception { cleanUp(); createSources(); @@ -819,7 +912,7 @@ public int execute(String tname) { try { return drv.run(qMap.get(tname)).getResponseCode(); } catch (CommandNeedRetryException e) { - // TODO Auto-generated catch block + System.out.println("driver failed to run the command: " + tname); e.printStackTrace(); return -1; } @@ -865,7 +958,7 @@ public void convertSequenceFileToTextFile() throws Exception { // Move all data from dest4_sequencefile to dest4 drv - .run("FROM dest4_sequencefile INSERT OVERWRITE TABLE dest4 SELECT dest4_sequencefile.*"); + .run("FROM dest4_sequencefile INSERT OVERWRITE TABLE dest4 SELECT dest4_sequencefile.*"); // Drop dest4_sequencefile db.dropTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, "dest4_sequencefile", @@ -1578,8 +1671,10 @@ public static boolean queryListRunnerMultiThreaded(File[] qfiles, QTestUtil[] qt } public static void outputTestFailureHelpMessage() { - System.err.println("See ./ql/target/tmp/log/hive.log or ./itests/qtest/target/tmp/log/hive.log, " - + "or check ./ql/target/surefire-reports or ./itests/qtest/target/surefire-reports/ for specific test cases logs."); + System.err.println( + "See ./ql/target/tmp/log/hive.log or ./itests/qtest/target/tmp/log/hive.log, or check " + + "./ql/target/surefire-reports or ./itests/qtest/target/surefire-reports/ for specific " + + "test cases logs."); System.err.flush(); } diff --git ql/src/test/queries/clientpositive/create_encrypted_table.q ql/src/test/queries/clientpositive/create_encrypted_table.q new file mode 100644 index 0000000..b411b11 --- /dev/null +++ ql/src/test/queries/clientpositive/create_encrypted_table.q @@ -0,0 +1,2 @@ +USE encryptedWith128BitsKeyDB; +CREATE TABLE a(k string, v string); diff --git shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java index 2e00d93..660e832 100644 --- shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java +++ shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java @@ -494,7 +494,7 @@ public FileSystem createProxyFileSystem(FileSystem fs, URI uri) { ret.put("HADOOPSPECULATIVEEXECREDUCERS", "mapred.reduce.tasks.speculative.execution"); ret.put("MAPREDSETUPCLEANUPNEEDED", "mapred.committer.job.setup.cleanup.needed"); ret.put("MAPREDTASKCLEANUPNEEDED", "mapreduce.job.committer.task.cleanup.needed"); - ret.put("HADOOPSECURITYKEYPROVIDER", "hadoop.encryption.is.not.supported"); + ret.put("HADOOPSECURITYKEYPROVIDER", "dfs.encryption.key.provider.uri"); return ret; } diff --git shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java index 8161fc1..a0598c7 100644 --- shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java +++ shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java @@ -24,8 +24,8 @@ import java.net.InetSocketAddress; import java.net.MalformedURLException; import java.net.URI; -import java.net.URISyntaxException; import java.security.AccessControlException; +import java.security.NoSuchAlgorithmException; import java.util.ArrayList; import java.util.Comparator; import java.util.HashMap; @@ -36,7 +36,7 @@ import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.crypto.key.KeyProvider; -import org.apache.hadoop.crypto.key.KeyProviderFactory; +import org.apache.hadoop.crypto.key.KeyProvider.Options; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.DefaultFileAccess; import org.apache.hadoop.fs.FSDataInputStream; @@ -96,7 +96,8 @@ public class Hadoop23Shims extends HadoopShimsSecure { HadoopShims.MiniDFSShim cluster = null; - + MiniDFSCluster miniDFSCluster = null; + KeyProvider keyProvider; final boolean zeroCopy; public Hadoop23Shims() { @@ -380,7 +381,9 @@ public void setupConfiguration(Configuration conf) { int numDataNodes, boolean format, String[] racks) throws IOException { - cluster = new MiniDFSShim(new MiniDFSCluster(conf, numDataNodes, format, racks)); + miniDFSCluster = new MiniDFSCluster(conf, numDataNodes, format, racks); + keyProvider = miniDFSCluster.getNameNode().getNamesystem().getProvider(); + cluster = new MiniDFSShim(miniDFSCluster); return cluster; } @@ -742,7 +745,7 @@ public FileSystem createProxyFileSystem(FileSystem fs, URI uri) { ret.put("HADOOPSPECULATIVEEXECREDUCERS", "mapreduce.reduce.speculative"); ret.put("MAPREDSETUPCLEANUPNEEDED", "mapreduce.job.committer.setup.cleanup.needed"); ret.put("MAPREDTASKCLEANUPNEEDED", "mapreduce.job.committer.task.cleanup.needed"); - ret.put("HADOOPSECURITYKEYPROVIDER", "hadoop.security.key.provider.path"); + ret.put("HADOOPSECURITYKEYPROVIDER", "dfs.encryption.key.provider.uri"); return ret; } @@ -938,12 +941,7 @@ public boolean runDistCp(Path src, Path dst, Configuration conf) throws IOExcept return (0 == rc); } - public static class HdfsEncryptionShim implements HadoopShims.HdfsEncryptionShim { - /** - * Gets information about key encryption metadata - */ - private KeyProvider keyProvider = null; - + public class HdfsEncryptionShim implements HadoopShims.HdfsEncryptionShim { /** * Gets information about HDFS encryption zones */ @@ -951,17 +949,6 @@ public boolean runDistCp(Path src, Path dst, Configuration conf) throws IOExcept public HdfsEncryptionShim(URI uri, Configuration conf) throws IOException { hdfsAdmin = new HdfsAdmin(uri, conf); - - try { - String keyProviderPath = conf.get(ShimLoader.getHadoopShims().getHadoopConfNames().get("HADOOPSECURITYKEYPROVIDER"), null); - if (keyProviderPath != null) { - keyProvider = KeyProviderFactory.get(new URI(keyProviderPath), conf); - } - } catch (URISyntaxException e) { - throw new IOException("Invalid HDFS security key provider path", e); - } catch (Exception e) { - throw new IOException("Cannot create HDFS security object: ", e); - } } @Override @@ -1003,6 +990,24 @@ public int comparePathKeyStrength(Path path1, Path path2) throws IOException { return compareKeyStrength(zone1.getKeyName(), zone2.getKeyName()); } + @Override + public void createEncryptionZone(Path path, String keyName) throws IOException { + hdfsAdmin.createEncryptionZone(path, keyName); + } + + @Override + public void createKey(String keyName, Configuration conf) + throws IOException, NoSuchAlgorithmException { + + if (keyProvider.getMetadata(keyName) != null) { + LOG.info("key " + keyName + " has already exists"); + return; + } + Options options = new Options(conf); + keyProvider.createKey(keyName, options); + keyProvider.flush(); + } + /** * Compares two encryption key strengths. * diff --git shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java index fa66a4a..8389f5f 100644 --- shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java +++ shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java @@ -26,6 +26,7 @@ import java.net.URISyntaxException; import java.nio.ByteBuffer; import java.security.AccessControlException; +import java.security.NoSuchAlgorithmException; import java.security.PrivilegedExceptionAction; import java.util.Comparator; import java.util.List; @@ -34,6 +35,7 @@ import javax.security.auth.login.LoginException; +import com.google.common.annotations.VisibleForTesting; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -400,7 +402,11 @@ public void refreshDefaultQueue(Configuration conf, String userName) /** * Verify proxy access to given UGI for given user - * @param ugi + * @param proxyUser + * @param realUserUgi + * @param ipAddress + * @param conf + * @throws IOException */ public void authorizeProxyAccess(String proxyUser, UserGroupInformation realUserUgi, String ipAddress, Configuration conf) throws IOException; @@ -819,6 +825,19 @@ public void checkFileAccess(FileSystem fs, FileStatus status, FsAction action) * @throws IOException If an error occurred attempting to get encryption/key metadata */ public int comparePathKeyStrength(Path path1, Path path2) throws IOException; + + /** + * create encryption zone by path and keyname + * @param path HDFS path to create encryption zone + * @param keyName keyname + * @throws IOException + */ + @VisibleForTesting + public void createEncryptionZone(Path path, String keyName) throws IOException; + + @VisibleForTesting + public void createKey(String keyName, Configuration conf) + throws IOException, NoSuchAlgorithmException; } /** @@ -842,6 +861,16 @@ public int comparePathKeyStrength(Path path1, Path path2) throws IOException { /* not supported */ return 0; } + + @Override + public void createEncryptionZone(Path path, String keyName) { + /* not supported */ + } + + @Override + public void createKey(String keyName, Configuration conf) { + /* not supported */ + } } /**