diff --git data/scripts/q_test_cleanup_for_encryption.sql data/scripts/q_test_cleanup_for_encryption.sql
new file mode 100644
index 0000000..d28f406
--- /dev/null
+++ data/scripts/q_test_cleanup_for_encryption.sql
@@ -0,0 +1,5 @@
+DROP DATABASE encryptedWith128BitsKeyDB;
+
+DROP DATABASE encryptedWith256BitsKeyDB;
+
+DROP DATABASE unencryptedDB;
\ No newline at end of file
diff --git data/scripts/q_test_init_for_encryption.sql data/scripts/q_test_init_for_encryption.sql
new file mode 100644
index 0000000..9245508
--- /dev/null
+++ data/scripts/q_test_init_for_encryption.sql
@@ -0,0 +1,5 @@
+CREATE DATABASE encryptedWith128BitsKeyDB;
+
+CREATE DATABASE encryptedWith256BitsKeyDB;
+
+CREATE DATABASE unencryptedDB;
\ No newline at end of file
diff --git itests/qtest/pom.xml itests/qtest/pom.xml
index 376f4a9..43c950f 100644
--- itests/qtest/pom.xml
+++ itests/qtest/pom.xml
@@ -532,6 +532,24 @@
hadoopVersion="${active.hadoop.version}"
initScript="q_test_init.sql"
cleanupScript="q_test_cleanup.sql"/>
+
+
diff --git itests/src/test/resources/testconfiguration.properties itests/src/test/resources/testconfiguration.properties
index 3ae001d..5bf24b5 100644
--- itests/src/test/resources/testconfiguration.properties
+++ itests/src/test/resources/testconfiguration.properties
@@ -271,6 +271,8 @@ minitez.query.files=bucket_map_join_tez1.q,\
tez_smb_1.q,\
vectorized_dynamic_partition_pruning.q
+encrypted.query.files.shared=
+
beeline.positive.exclude=add_part_exist.q,\
alter1.q,\
alter2.q,\
diff --git itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
index 31d5c29..ef883fe 100644
--- itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+++ itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -39,6 +39,7 @@
import java.io.Serializable;
import java.io.StringWriter;
import java.net.URL;
+import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -76,6 +77,7 @@
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.lockmgr.zookeeper.ZooKeeperHiveLockManager;
import org.apache.hadoop.hive.ql.metadata.Hive;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.parse.ASTNode;
import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer;
@@ -102,6 +104,24 @@
public class QTestUtil {
public static final String UTF_8 = "UTF-8";
+
+ // database names used for testing the encrypted databases
+ private static final String ENCRYPTED_WITH_128_BITS_KEY_DB_NAME = "encryptedwith128bitskeydb";
+ private static final String ENCRYPTED_WITH_256_BITS_KEY_DB_NAME = "encryptedwith256bitskeydb";
+ private static final String UNENCRYPTED_DB_NAME = "unencrypteddb";
+
+ // security property names
+ private static final String SECURITY_KEY_BIT_LENGTH_PROP_NAME =
+ "hadoop.security.key.default.bitlength";
+ private static final String SECURITY_KEY_CIPHER_NAME = "hadoop.security.key.default.cipher";
+
+ // keyNames used for encrypting the hdfs path
+ private final String KEY_NAME_IN_128 = "k128";
+ private final String KEY_NAME_IN_256 = "k256";
+
+ // hadoop cipher
+ private final String HADOOP_CIPHER_NAME = "AES/CTR/NoPadding";
+
private static final Log LOG = LogFactory.getLog("QTestUtil");
private static final String QTEST_LEAVE_FILES = "QTEST_LEAVE_FILES";
private final String defaultInitScript = "q_test_init.sql";
@@ -130,6 +150,7 @@
private CliDriver cliDriver;
private HadoopShims.MiniMrShim mr = null;
private HadoopShims.MiniDFSShim dfs = null;
+ private HadoopShims.HdfsEncryptionShim hes = null;
private boolean miniMr = false;
private String hadoopVer = null;
private QTestSetup setup = null;
@@ -245,6 +266,13 @@ private String getHadoopMainVersion(String input) {
return null;
}
+ private void initEncryptionRelatedConf() {
+ HadoopShims shims = ShimLoader.getHadoopShims();
+ // set up the java key provider for encrypted hdfs cluster
+ conf.set(shims.getHadoopConfNames().get("HADOOPSECURITYKEYPROVIDER"), getKeyProviderURI());
+ conf.set(SECURITY_KEY_CIPHER_NAME, HADOOP_CIPHER_NAME);
+ }
+
public void initConf() throws Exception {
String vectorizationEnabled = System.getProperty("test.vectorization.enabled");
@@ -280,6 +308,7 @@ public void initConf() throws Exception {
public enum MiniClusterType {
mr,
tez,
+ encrypted,
none;
public static MiniClusterType valueForString(String type) {
@@ -287,6 +316,8 @@ public static MiniClusterType valueForString(String type) {
return mr;
} else if (type.equals("tez")) {
return tez;
+ } else if (type.equals("encrypted")) {
+ return encrypted;
} else {
return none;
}
@@ -299,6 +330,15 @@ public QTestUtil(String outDir, String logDir, MiniClusterType clusterType, Stri
this(outDir, logDir, clusterType, null, hadoopVer, initScript, cleanupScript);
}
+ private String getKeyProviderURI() {
+ // Use the target directory if it is not specified
+ String HIVE_ROOT = QTestUtil.ensurePathEndsInSlash(System.getProperty("hive.root"));
+ String keyDir = HIVE_ROOT + "ql/target/";
+
+ // put the jks file in the current test path only for test purpose
+ return "jceks://file" + new Path(keyDir, "test.jks").toUri();
+ }
+
public QTestUtil(String outDir, String logDir, MiniClusterType clusterType,
String confDir, String hadoopVer, String initScript, String cleanupScript)
throws Exception {
@@ -323,8 +363,21 @@ public QTestUtil(String outDir, String logDir, MiniClusterType clusterType,
int numberOfDataNodes = 4;
if (clusterType != MiniClusterType.none) {
- dfs = shims.getMiniDfs(conf, numberOfDataNodes, true, null);
- FileSystem fs = dfs.getFileSystem();
+ FileSystem fs;
+
+ if (clusterType == MiniClusterType.encrypted) {
+ initEncryptionRelatedConf();
+
+ dfs = shims.getMiniDfs(conf, numberOfDataNodes, true, null);
+ fs = dfs.getFileSystem();
+ // set up the java key provider for encrypted hdfs cluster
+ hes = shims.createHdfsEncryptionShim(fs, conf);
+ LOG.info("key provider is initialized");
+ } else {
+ dfs = shims.getMiniDfs(conf, numberOfDataNodes, true, null);
+ fs = dfs.getFileSystem();
+ }
+
String uriString = WindowsPathUtil.getHdfsUriString(fs.getUri().toString());
if (clusterType == MiniClusterType.tez) {
mr = shims.getMiniTezCluster(conf, 4, uriString, 1);
@@ -340,7 +393,6 @@ public QTestUtil(String outDir, String logDir, MiniClusterType clusterType,
if (dataDir == null) {
dataDir = new File(".").getAbsolutePath() + "/data/files";
}
-
testFiles = dataDir;
// Use the current directory if it is not specified
@@ -368,7 +420,7 @@ public void shutdown() throws Exception {
if (System.getenv(QTEST_LEAVE_FILES) == null) {
cleanUp();
}
-
+
setup.tearDown();
if (mr != null) {
mr.shutdown();
@@ -538,6 +590,19 @@ public void clearPostTestEffects() throws Exception {
}
/**
+ * For the security type, we should reserve the encrypted databases for the test purpose
+ */
+ private boolean checkDBIfNeedToBePreserved(String dbName) {
+ if (clusterType == MiniClusterType.encrypted) {
+ return (DEFAULT_DATABASE_NAME.equals(dbName) ||
+ ENCRYPTED_WITH_128_BITS_KEY_DB_NAME.equals(dbName) ||
+ ENCRYPTED_WITH_256_BITS_KEY_DB_NAME.equals(dbName) || UNENCRYPTED_DB_NAME.equals(dbName));
+ } else {
+ return DEFAULT_DATABASE_NAME.equals(dbName);
+ }
+ }
+
+ /**
* Clear out any side effects of running tests
*/
public void clearTestSideEffects() throws Exception {
@@ -545,11 +610,11 @@ public void clearTestSideEffects() throws Exception {
return;
}
// Delete any tables other than the source tables
- // and any databases other than the default database.
+ // and any databases other than the default database or encrypted dbs in encryption mode.
for (String dbName : db.getAllDatabases()) {
SessionState.get().setCurrentDatabase(dbName);
for (String tblName : db.getAllTables()) {
- if (!DEFAULT_DATABASE_NAME.equals(dbName) || !srcTables.contains(tblName)) {
+ if (!checkDBIfNeedToBePreserved(dbName) || !srcTables.contains(tblName)) {
Table tblObj = db.getTable(tblName);
// dropping index table can not be dropped directly. Dropping the base
// table will automatically drop all its index table
@@ -567,7 +632,7 @@ public void clearTestSideEffects() throws Exception {
}
}
}
- if (!DEFAULT_DATABASE_NAME.equals(dbName)) {
+ if (!checkDBIfNeedToBePreserved(dbName)) {
// Drop cascade, may need to drop functions
db.dropDatabase(dbName, true, true, true);
}
@@ -593,11 +658,15 @@ public void clearTestSideEffects() throws Exception {
db.dropRole(roleName);
}
}
- // allocate and initialize a new conf since a test can
- // modify conf by using 'set' commands
- conf = new HiveConf (Driver.class);
- initConf();
- db = Hive.get(conf); // propagate new conf to meta store
+
+ if (clusterType != MiniClusterType.encrypted) {
+ // allocate and initialize a new conf since a test can
+ // modify conf by using 'set' commands
+ conf = new HiveConf (Driver.class);
+ initConf();
+ // renew the metastore since the cluster type is unencrypted
+ db = Hive.get(conf); // propagate new conf to meta store
+ }
setup.preTest(conf);
}
@@ -685,6 +754,10 @@ public void createSources() throws Exception {
cliDriver.processLine(initCommands);
conf.setBoolean("hive.test.init.phase", false);
+
+ if (clusterType == MiniClusterType.encrypted) {
+ initEncryptionZone();
+ }
}
public void init() throws Exception {
@@ -705,6 +778,29 @@ public void init() throws Exception {
sem = new SemanticAnalyzer(conf);
}
+ private void initEncryptionZone() throws IOException, NoSuchAlgorithmException, HiveException {
+ // current only aes/ctr/nopadding cipher is supported
+ conf.set(SECURITY_KEY_CIPHER_NAME, HADOOP_CIPHER_NAME);
+
+ // create encryption zone via a 128-bits key respectively for encrypted database 1
+ conf.set(SECURITY_KEY_BIT_LENGTH_PROP_NAME, "128");
+
+ hes.createKey(KEY_NAME_IN_128, conf);
+ hes.createEncryptionZone(
+ new Path(db.getDatabase(ENCRYPTED_WITH_128_BITS_KEY_DB_NAME).getLocationUri()),
+ KEY_NAME_IN_128);
+
+ // create encryption zone via a 256-bits key respectively for encrypted database 2
+ conf.set(SECURITY_KEY_BIT_LENGTH_PROP_NAME, "256");
+
+ // AES-256 can be used only if JCE is installed in your environment. Otherwise, any encryption
+ // with this key will fail. Keys can be created, but when you try to encrypt something, fails.
+ hes.createKey(KEY_NAME_IN_256, conf);
+ hes.createEncryptionZone(
+ new Path(db.getDatabase(ENCRYPTED_WITH_256_BITS_KEY_DB_NAME).getLocationUri()),
+ KEY_NAME_IN_256);
+ }
+
public void init(String tname) throws Exception {
cleanUp();
createSources();
@@ -819,7 +915,7 @@ public int execute(String tname) {
try {
return drv.run(qMap.get(tname)).getResponseCode();
} catch (CommandNeedRetryException e) {
- // TODO Auto-generated catch block
+ LOG.error("driver failed to run the command: " + tname + " due to the exception: ", e);
e.printStackTrace();
return -1;
}
@@ -865,7 +961,7 @@ public void convertSequenceFileToTextFile() throws Exception {
// Move all data from dest4_sequencefile to dest4
drv
- .run("FROM dest4_sequencefile INSERT OVERWRITE TABLE dest4 SELECT dest4_sequencefile.*");
+ .run("FROM dest4_sequencefile INSERT OVERWRITE TABLE dest4 SELECT dest4_sequencefile.*");
// Drop dest4_sequencefile
db.dropTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, "dest4_sequencefile",
@@ -1578,8 +1674,10 @@ public static boolean queryListRunnerMultiThreaded(File[] qfiles, QTestUtil[] qt
}
public static void outputTestFailureHelpMessage() {
- System.err.println("See ./ql/target/tmp/log/hive.log or ./itests/qtest/target/tmp/log/hive.log, "
- + "or check ./ql/target/surefire-reports or ./itests/qtest/target/surefire-reports/ for specific test cases logs.");
+ System.err.println(
+ "See ./ql/target/tmp/log/hive.log or ./itests/qtest/target/tmp/log/hive.log, or check " +
+ "./ql/target/surefire-reports or ./itests/qtest/target/surefire-reports/ for specific " +
+ "test cases logs.");
System.err.flush();
}
diff --git shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
index 2e00d93..660e832 100644
--- shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
+++ shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
@@ -494,7 +494,7 @@ public FileSystem createProxyFileSystem(FileSystem fs, URI uri) {
ret.put("HADOOPSPECULATIVEEXECREDUCERS", "mapred.reduce.tasks.speculative.execution");
ret.put("MAPREDSETUPCLEANUPNEEDED", "mapred.committer.job.setup.cleanup.needed");
ret.put("MAPREDTASKCLEANUPNEEDED", "mapreduce.job.committer.task.cleanup.needed");
- ret.put("HADOOPSECURITYKEYPROVIDER", "hadoop.encryption.is.not.supported");
+ ret.put("HADOOPSECURITYKEYPROVIDER", "dfs.encryption.key.provider.uri");
return ret;
}
diff --git shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
index 8161fc1..7b0f884 100644
--- shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
+++ shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
@@ -26,6 +26,7 @@
import java.net.URI;
import java.net.URISyntaxException;
import java.security.AccessControlException;
+import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
@@ -36,6 +37,7 @@
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.crypto.key.KeyProvider;
+import org.apache.hadoop.crypto.key.KeyProvider.Options;
import org.apache.hadoop.crypto.key.KeyProviderFactory;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.DefaultFileAccess;
@@ -96,7 +98,8 @@
public class Hadoop23Shims extends HadoopShimsSecure {
HadoopShims.MiniDFSShim cluster = null;
-
+ MiniDFSCluster miniDFSCluster = null;
+ KeyProvider keyProvider;
final boolean zeroCopy;
public Hadoop23Shims() {
@@ -380,7 +383,9 @@ public void setupConfiguration(Configuration conf) {
int numDataNodes,
boolean format,
String[] racks) throws IOException {
- cluster = new MiniDFSShim(new MiniDFSCluster(conf, numDataNodes, format, racks));
+ miniDFSCluster = new MiniDFSCluster(conf, numDataNodes, format, racks);
+ keyProvider = miniDFSCluster.getNameNode().getNamesystem().getProvider();
+ cluster = new MiniDFSShim(miniDFSCluster);
return cluster;
}
@@ -742,7 +747,7 @@ public FileSystem createProxyFileSystem(FileSystem fs, URI uri) {
ret.put("HADOOPSPECULATIVEEXECREDUCERS", "mapreduce.reduce.speculative");
ret.put("MAPREDSETUPCLEANUPNEEDED", "mapreduce.job.committer.setup.cleanup.needed");
ret.put("MAPREDTASKCLEANUPNEEDED", "mapreduce.job.committer.task.cleanup.needed");
- ret.put("HADOOPSECURITYKEYPROVIDER", "hadoop.security.key.provider.path");
+ ret.put("HADOOPSECURITYKEYPROVIDER", "dfs.encryption.key.provider.uri");
return ret;
}
@@ -938,12 +943,7 @@ public boolean runDistCp(Path src, Path dst, Configuration conf) throws IOExcept
return (0 == rc);
}
- public static class HdfsEncryptionShim implements HadoopShims.HdfsEncryptionShim {
- /**
- * Gets information about key encryption metadata
- */
- private KeyProvider keyProvider = null;
-
+ public class HdfsEncryptionShim implements HadoopShims.HdfsEncryptionShim {
/**
* Gets information about HDFS encryption zones
*/
@@ -951,16 +951,21 @@ public boolean runDistCp(Path src, Path dst, Configuration conf) throws IOExcept
public HdfsEncryptionShim(URI uri, Configuration conf) throws IOException {
hdfsAdmin = new HdfsAdmin(uri, conf);
-
- try {
- String keyProviderPath = conf.get(ShimLoader.getHadoopShims().getHadoopConfNames().get("HADOOPSECURITYKEYPROVIDER"), null);
- if (keyProviderPath != null) {
- keyProvider = KeyProviderFactory.get(new URI(keyProviderPath), conf);
+ // We get the key provider via the MiniDFSCluster in the test and in the product
+ // environment we get the key provider via the key provider factory.
+ if (keyProvider == null) {
+ try {
+ String keyProviderPath = conf
+ .get(ShimLoader.getHadoopShims().getHadoopConfNames().get("HADOOPSECURITYKEYPROVIDER"),
+ null);
+ if (keyProviderPath != null) {
+ keyProvider = KeyProviderFactory.get(new URI(keyProviderPath), conf);
+ }
+ } catch (URISyntaxException e) {
+ throw new IOException("Invalid HDFS security key provider path", e);
+ } catch (Exception e) {
+ throw new IOException("Cannot create HDFS security object: ", e);
}
- } catch (URISyntaxException e) {
- throw new IOException("Invalid HDFS security key provider path", e);
- } catch (Exception e) {
- throw new IOException("Cannot create HDFS security object: ", e);
}
}
@@ -1003,6 +1008,24 @@ public int comparePathKeyStrength(Path path1, Path path2) throws IOException {
return compareKeyStrength(zone1.getKeyName(), zone2.getKeyName());
}
+ @Override
+ public void createEncryptionZone(Path path, String keyName) throws IOException {
+ hdfsAdmin.createEncryptionZone(path, keyName);
+ }
+
+ @Override
+ public void createKey(String keyName, Configuration conf)
+ throws IOException, NoSuchAlgorithmException {
+
+ if (keyProvider.getMetadata(keyName) != null) {
+ LOG.info("key " + keyName + " has already exists");
+ return;
+ }
+ Options options = new Options(conf);
+ keyProvider.createKey(keyName, options);
+ keyProvider.flush();
+ }
+
/**
* Compares two encryption key strengths.
*
diff --git shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
index fa66a4a..8389f5f 100644
--- shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
+++ shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
@@ -26,6 +26,7 @@
import java.net.URISyntaxException;
import java.nio.ByteBuffer;
import java.security.AccessControlException;
+import java.security.NoSuchAlgorithmException;
import java.security.PrivilegedExceptionAction;
import java.util.Comparator;
import java.util.List;
@@ -34,6 +35,7 @@
import javax.security.auth.login.LoginException;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -400,7 +402,11 @@ public void refreshDefaultQueue(Configuration conf, String userName)
/**
* Verify proxy access to given UGI for given user
- * @param ugi
+ * @param proxyUser
+ * @param realUserUgi
+ * @param ipAddress
+ * @param conf
+ * @throws IOException
*/
public void authorizeProxyAccess(String proxyUser, UserGroupInformation realUserUgi,
String ipAddress, Configuration conf) throws IOException;
@@ -819,6 +825,19 @@ public void checkFileAccess(FileSystem fs, FileStatus status, FsAction action)
* @throws IOException If an error occurred attempting to get encryption/key metadata
*/
public int comparePathKeyStrength(Path path1, Path path2) throws IOException;
+
+ /**
+ * create encryption zone by path and keyname
+ * @param path HDFS path to create encryption zone
+ * @param keyName keyname
+ * @throws IOException
+ */
+ @VisibleForTesting
+ public void createEncryptionZone(Path path, String keyName) throws IOException;
+
+ @VisibleForTesting
+ public void createKey(String keyName, Configuration conf)
+ throws IOException, NoSuchAlgorithmException;
}
/**
@@ -842,6 +861,16 @@ public int comparePathKeyStrength(Path path1, Path path2) throws IOException {
/* not supported */
return 0;
}
+
+ @Override
+ public void createEncryptionZone(Path path, String keyName) {
+ /* not supported */
+ }
+
+ @Override
+ public void createKey(String keyName, Configuration conf) {
+ /* not supported */
+ }
}
/**