From aefc0eeb85c3c958c1213fde2c4736606e4e2520 Mon Sep 17 00:00:00 2001 From: Ashutosh Chauhan Date: Wed, 3 Feb 2016 13:50:12 -0800 Subject: [PATCH] HIVE-13004 : Remove encryption shims --- .../hcatalog/pig/TestHCatLoaderEncryption.java | 10 +- .../java/org/apache/hadoop/hive/ql/QTestUtil.java | 17 +- .../hadoop/hive/metastore/HiveMetaStore.java | 11 +- .../org/apache/hadoop/hive/ql/metadata/Hive.java | 11 +- .../hadoop/hive/ql/parse/SemanticAnalyzer.java | 33 ++-- .../ql/processors/CommandProcessorFactory.java | 6 +- .../hadoop/hive/ql/processors/CryptoProcessor.java | 32 ++-- .../hadoop/hive/ql/session/SessionState.java | 25 --- .../apache/hadoop/hive/shims/Hadoop23Shims.java | 180 --------------------- .../java/org/apache/hadoop/hive/io/HdfsUtils.java | 164 ++++++++++++++++++- .../org/apache/hadoop/hive/shims/HadoopShims.java | 119 -------------- 11 files changed, 202 insertions(+), 406 deletions(-) diff --git a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderEncryption.java b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderEncryption.java index 676a4ed..1902693 100644 --- a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderEncryption.java +++ b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderEncryption.java @@ -95,7 +95,6 @@ private static final String SECURITY_KEY_PROVIDER_URI_NAME = "dfs.encryption.key.provider.uri"; private HadoopShims.MiniDFSShim dfs = null; - private HadoopShims.HdfsEncryptionShim hes = null; private final String[] testOnlyCommands = new String[]{"crypto"}; private Driver driver; private Map> basicInputData; @@ -111,7 +110,7 @@ }}); }}; - private String storageFormat; + private final String storageFormat; @Parameterized.Parameters public static Collection generateParameters() { @@ -180,7 +179,7 @@ public void setup() throws Exception { String s = hiveConf.get("hdfs.minidfs.basedir"); if(s == null || s.length() <= 0) { //return System.getProperty("test.build.data", "build/test/data") + "/dfs/"; - hiveConf.set("hdfs.minidfs.basedir", + hiveConf.set("hdfs.minidfs.basedir", System.getProperty("test.build.data", "build/test/data") + "_" + System.currentTimeMillis() + "_" + salt.getAndIncrement() + "/dfs/"); } @@ -225,16 +224,11 @@ public void setup() throws Exception { } void initEncryptionShim(HiveConf conf) throws IOException { - FileSystem fs; HadoopShims shims = ShimLoader.getHadoopShims(); conf.set(SECURITY_KEY_PROVIDER_URI_NAME, getKeyProviderURI()); WindowsPathUtil.convertPathsFromWindowsToHdfs(conf); int numberOfDataNodes = 4; dfs = shims.getMiniDfs(conf, numberOfDataNodes, true, null); - fs = dfs.getFileSystem(); - - // set up a java key provider for encrypted hdfs cluster - hes = shims.createHdfsEncryptionShim(fs, conf); } public static String ensurePathEndsInSlash(String path) { diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java index 5e81e98..93d0a68 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java @@ -74,6 +74,7 @@ import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster; +import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hive.cli.CliDriver; import org.apache.hadoop.hive.cli.CliSessionState; import org.apache.hadoop.hive.common.io.CachingPrintStream; @@ -82,6 +83,7 @@ import org.apache.hadoop.hive.common.io.SortPrintStream; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.io.HdfsUtils; import org.apache.hadoop.hive.llap.io.api.LlapProxy; import org.apache.hadoop.hive.metastore.MetaStoreUtils; import org.apache.hadoop.hive.metastore.api.Index; @@ -165,7 +167,6 @@ private CliDriver cliDriver; private HadoopShims.MiniMrShim mr = null; private HadoopShims.MiniDFSShim dfs = null; - private HadoopShims.HdfsEncryptionShim hes = null; private String hadoopVer = null; private QTestSetup setup = null; private TezSessionState tezSessionState = null; @@ -413,11 +414,6 @@ public QTestUtil(String outDir, String logDir, MiniClusterType clusterType, dfs = shims.getMiniDfs(conf, numberOfDataNodes, true, null); fs = dfs.getFileSystem(); - - // set up the java key provider for encrypted hdfs cluster - hes = shims.createHdfsEncryptionShim(fs, conf); - - LOG.info("key provider is initialized"); } else { dfs = shims.getMiniDfs(conf, numberOfDataNodes, true, null); fs = dfs.getFileSystem(); @@ -706,12 +702,9 @@ public void clearPostTestEffects() throws Exception { } public void clearKeysCreatedInTests() { - if (hes == null) { - return; - } try { - for (String keyAlias : hes.getKeys()) { - hes.deleteKey(keyAlias); + for (String keyAlias : ((DistributedFileSystem) FileSystem.get(conf)).getClient().getKeyProvider().getKeys()) { + HdfsUtils.deleteKey(keyAlias, conf); } } catch (IOException e) { LOG.error("Fail to clean the keys created in test due to the error", e); @@ -2136,7 +2129,7 @@ public int compare(String str1, String str2) { int i = 0; while (!statements.isEmpty()) { // PreparedStatement extend Statement - Statement st = (Statement)statements.remove(i); + Statement st = statements.remove(i); try { if (st != null) { st.close(); diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index 98fbf70..7d177b7 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -26,6 +26,7 @@ import com.google.common.collect.ImmutableListMultimap; import com.google.common.collect.Lists; import com.google.common.collect.Multimaps; + import org.apache.commons.cli.OptionBuilder; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; @@ -43,6 +44,7 @@ import org.apache.hadoop.hive.common.metrics.common.MetricsVariable; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.io.HdfsUtils; import org.apache.hadoop.hive.metastore.api.*; import org.apache.hadoop.hive.metastore.events.AddIndexEvent; import org.apache.hadoop.hive.metastore.events.AddPartitionEvent; @@ -110,6 +112,7 @@ import org.slf4j.LoggerFactory; import javax.jdo.JDOException; + import java.io.IOException; import java.nio.ByteBuffer; import java.text.DateFormat; @@ -458,7 +461,7 @@ public Object getValue() { } expressionProxy = PartFilterExprUtil.createExpressionProxy(hiveConf); - fileMetadataManager = new FileMetadataManager((ThreadLocalRawStore)this, hiveConf); + fileMetadataManager = new FileMetadataManager(this, hiveConf); } private String addPrefix(String s) { @@ -1550,9 +1553,7 @@ private void checkTrashPurgeCombination(Path pathToData, String objectName, bool if (trashEnabled) { try { - HadoopShims.HdfsEncryptionShim shim = - ShimLoader.getHadoopShims().createHdfsEncryptionShim(FileSystem.get(hiveConf), hiveConf); - if (shim.isPathEncrypted(pathToData)) { + if (HdfsUtils.isPathEncrypted(pathToData, hiveConf)) { throw new MetaException("Unable to drop " + objectName + " because it is in an encryption zone" + " and trash is enabled. Use PURGE option to skip trash."); } @@ -6235,7 +6236,7 @@ public static void startMetaStore(int port, HadoopThriftAuthBridge bridge, LOG.info("Starting DB backed MetaStore Server"); } } - + TServerTransport serverTransport = tcpKeepAlive ? new TServerSocketKeepAlive(port) : new TServerSocket(port); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index a92c002..916ecaa 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -45,6 +45,7 @@ import java.util.concurrent.atomic.AtomicInteger; import com.google.common.collect.ImmutableMap; + import javax.jdo.JDODataStoreException; import org.apache.hadoop.conf.Configuration; @@ -59,6 +60,7 @@ import org.apache.hadoop.hive.common.classification.InterfaceStability.Unstable; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.io.HdfsUtils; import org.apache.hadoop.hive.metastore.HiveMetaException; import org.apache.hadoop.hive.metastore.HiveMetaHook; import org.apache.hadoop.hive.metastore.HiveMetaHookLoader; @@ -2746,7 +2748,7 @@ public static boolean moveFile(HiveConf conf, Path srcf, Path destf, destFs.copyFromLocalFile(srcf, destf); success = true; } else { - if (needToCopy(srcf, destf, srcFs, destFs)) { + if (needToCopy(srcf, destf, srcFs, destFs, conf)) { //copy if across file system or encryption zones. LOG.info("Copying source " + srcf + " to " + destf + " because HDFS encryption zones are different."); success = FileUtils.copy(srcf.getFileSystem(conf), srcf, destf.getFileSystem(conf), destf, @@ -2809,16 +2811,15 @@ public static boolean moveFile(HiveConf conf, Path srcf, Path destf, * If moving across different FileSystems or differnent encryption zone, need to do a File copy instead of rename. * TODO- consider if need to do this for different file authority. */ - static protected boolean needToCopy(Path srcf, Path destf, FileSystem srcFs, FileSystem destFs) throws HiveException, IOException { + static protected boolean needToCopy(Path srcf, Path destf, FileSystem srcFs, FileSystem destFs, HiveConf conf) throws HiveException, IOException { //Check if different FileSystems if (!srcFs.getClass().equals(destFs.getClass())) { return true; } //Check if different encryption zones - HadoopShims.HdfsEncryptionShim hdfsEncryptionShim = SessionState.get().getHdfsEncryptionShim(); - return hdfsEncryptionShim != null && (hdfsEncryptionShim.isPathEncrypted(srcf) || hdfsEncryptionShim.isPathEncrypted(destf)) - && !hdfsEncryptionShim.arePathsOnSameEncryptionZone(srcf, destf); + return (HdfsUtils.isPathEncrypted(srcf, conf) || HdfsUtils.isPathEncrypted(destf,conf)) + && !HdfsUtils.arePathsOnSameEncryptionZone(srcf, destf, conf); } /** diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index ba1945f..4dad22f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -59,6 +59,7 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.conf.HiveConf.StrictChecks; +import org.apache.hadoop.hive.io.HdfsUtils; import org.apache.hadoop.hive.metastore.MetaStoreUtils; import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.Warehouse; @@ -1870,20 +1871,11 @@ public void getMetaData(QB qb, ReadEntity parentInput) throws SemanticException * @throws HiveException If an error occurs while checking for encryption */ private boolean isPathEncrypted(Path path) throws HiveException { - HadoopShims.HdfsEncryptionShim hdfsEncryptionShim; - - hdfsEncryptionShim = SessionState.get().getHdfsEncryptionShim(); - if (hdfsEncryptionShim != null) { - try { - if (hdfsEncryptionShim.isPathEncrypted(path)) { - return true; - } - } catch (Exception e) { - throw new HiveException("Unable to determine if " + path + " is encrypted: " + e, e); - } + try { + return HdfsUtils.isPathEncrypted(path, conf); + } catch (Exception e) { + throw new HiveException("Unable to determine if " + path + " is encrypted: " + e, e); } - - return false; } /** @@ -1895,18 +1887,11 @@ private boolean isPathEncrypted(Path path) throws HiveException { * @throws HiveException If an error occurs while comparing key strengths. */ private int comparePathKeyStrength(Path p1, Path p2) throws HiveException { - HadoopShims.HdfsEncryptionShim hdfsEncryptionShim; - - hdfsEncryptionShim = SessionState.get().getHdfsEncryptionShim(); - if (hdfsEncryptionShim != null) { - try { - return hdfsEncryptionShim.comparePathKeyStrength(p1, p2); - } catch (Exception e) { - throw new HiveException("Unable to compare key strength for " + p1 + " and " + p2 + " : " + e, e); - } + try { + return HdfsUtils.comparePathKeyStrength(p1, p2, conf); + } catch (IOException e) { + throw new HiveException("Unable to compare key strength for " + p1 + " and " + p2 + " : " + e, e); } - - return 0; // Non-encrypted path (or equals strength) } /** diff --git a/ql/src/java/org/apache/hadoop/hive/ql/processors/CommandProcessorFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/processors/CommandProcessorFactory.java index c343b0e..b77bd29 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/processors/CommandProcessorFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/processors/CommandProcessorFactory.java @@ -93,11 +93,7 @@ public static CommandProcessor getForHiveCommandInternal(String[] cmd, HiveConf case RELOAD: return new ReloadProcessor(); case CRYPTO: - try { - return new CryptoProcessor(SessionState.get().getHdfsEncryptionShim(), conf); - } catch (HiveException e) { - throw new SQLException("Fail to start the command processor due to the exception: ", e); - } + return new CryptoProcessor(conf); default: throw new AssertionError("Unknown HiveCommand " + hiveCommand); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/processors/CryptoProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/processors/CryptoProcessor.java index 1acdc95..0396c9f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/processors/CryptoProcessor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/processors/CryptoProcessor.java @@ -28,9 +28,9 @@ import org.slf4j.LoggerFactory; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.io.HdfsUtils; import org.apache.hadoop.hive.ql.CommandNeedRetryException; import org.apache.hadoop.hive.ql.session.SessionState; -import org.apache.hadoop.hive.shims.HadoopShims; import java.io.IOException; import java.util.Arrays; @@ -42,18 +42,15 @@ public class CryptoProcessor implements CommandProcessor { public static final Logger LOG = LoggerFactory.getLogger(CryptoProcessor.class.getName()); - private HadoopShims.HdfsEncryptionShim encryptionShim; + private final Options CREATE_KEY_OPTIONS; + private final Options DELETE_KEY_OPTIONS; + private final Options CREATE_ZONE_OPTIONS; - private Options CREATE_KEY_OPTIONS; - private Options DELETE_KEY_OPTIONS; - private Options CREATE_ZONE_OPTIONS; + private final int DEFAULT_BIT_LENGTH = 128; - private int DEFAULT_BIT_LENGTH = 128; + private final HiveConf conf; - private HiveConf conf; - - public CryptoProcessor(HadoopShims.HdfsEncryptionShim encryptionShim, HiveConf conf) { - this.encryptionShim = encryptionShim; + public CryptoProcessor (HiveConf conf) { this.conf = conf; CREATE_KEY_OPTIONS = new Options(); @@ -93,10 +90,6 @@ public CommandProcessorResponse run(String command) throws CommandNeedRetryExcep return returnErrorResponse("Command arguments are empty."); } - if (encryptionShim == null) { - return returnErrorResponse("Hadoop encryption shim is not initialized."); - } - String action = args[0]; String params[] = Arrays.copyOfRange(args, 1, args.length); @@ -130,7 +123,7 @@ private void createEncryptionKey(String[] params) throws Exception { String bitLength = args.getOptionValue("bitLength", Integer.toString(DEFAULT_BIT_LENGTH)); try { - encryptionShim.createKey(keyName, new Integer(bitLength)); + HdfsUtils.createKey(keyName, new Integer(bitLength), conf); } catch (Exception e) { throw new Exception("Cannot create encryption key: " + e.getMessage()); } @@ -149,13 +142,8 @@ private void createEncryptionZone(String[] params) throws Exception { String keyName = args.getOptionValue("keyName"); Path cryptoZone = new Path(args.getOptionValue("path")); - if (cryptoZone == null) { - throw new Exception("Cannot create encryption zone: Invalid path '" - + args.getOptionValue("path") + "'"); - } - try { - encryptionShim.createEncryptionZone(cryptoZone, keyName); + HdfsUtils.createEncryptionZone(cryptoZone, keyName, conf); } catch (IOException e) { throw new Exception("Cannot create encryption zone: " + e.getMessage()); } @@ -174,7 +162,7 @@ private void deleteEncryptionKey(String[] params) throws Exception { String keyName = args.getOptionValue("keyName"); try { - encryptionShim.deleteKey(keyName); + HdfsUtils.deleteKey(keyName, conf); } catch (IOException e) { throw new Exception("Cannot delete encryption key: " + e.getMessage()); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java index 2a1458e..b1b5594 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java @@ -42,7 +42,6 @@ import java.util.concurrent.locks.ReentrantLock; import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; @@ -83,8 +82,6 @@ import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthzSessionContext.CLIENT_TYPE; import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveMetastoreClientFactoryImpl; import org.apache.hadoop.hive.ql.util.ResourceDownloader; -import org.apache.hadoop.hive.shims.HadoopShims; -import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.hive.shims.Utils; import org.apache.hadoop.security.UserGroupInformation; @@ -211,11 +208,6 @@ private SparkSession sparkSession; /** - * Gets information about HDFS encryption - */ - private HadoopShims.HdfsEncryptionShim hdfsEncryptionShim; - - /** * Lineage state. */ LineageState ls; @@ -424,23 +416,6 @@ public HiveTxnManager getTxnMgr() { return txnMgr; } - public HadoopShims.HdfsEncryptionShim getHdfsEncryptionShim() throws HiveException { - if (hdfsEncryptionShim == null) { - try { - FileSystem fs = FileSystem.get(conf); - if ("hdfs".equals(fs.getUri().getScheme())) { - hdfsEncryptionShim = ShimLoader.getHadoopShims().createHdfsEncryptionShim(fs, conf); - } else { - LOG.info("Could not get hdfsEncryptionShim, it is only applicable to hdfs filesystem."); - } - } catch (Exception e) { - throw new HiveException(e); - } - } - - return hdfsEncryptionShim; - } - // SessionState is not available in runtime and Hive.get().getConf() is not safe to call private static class SessionStates { private SessionState state; diff --git a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java index 31060a2..6008489 100644 --- a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java +++ b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java @@ -27,7 +27,6 @@ import java.net.URI; import java.nio.ByteBuffer; import java.security.AccessControlException; -import java.security.NoSuchAlgorithmException; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; @@ -39,7 +38,6 @@ import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.crypto.key.KeyProvider; -import org.apache.hadoop.crypto.key.KeyProvider.Options; import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.DefaultFileAccess; @@ -64,9 +62,7 @@ import org.apache.hadoop.hdfs.DFSClient; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.hdfs.client.HdfsAdmin; import org.apache.hadoop.hdfs.protocol.DirectoryListing; -import org.apache.hadoop.hdfs.protocol.EncryptionZone; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus; import org.apache.hadoop.io.LongWritable; @@ -1280,182 +1276,6 @@ public boolean runDistCp(Path src, Path dst, Configuration conf) throws IOExcept } } - private static Boolean hdfsEncryptionSupport; - - public static boolean isHdfsEncryptionSupported() { - if (hdfsEncryptionSupport == null) { - Method m = null; - - try { - m = HdfsAdmin.class.getMethod("getEncryptionZoneForPath", Path.class); - } catch (NoSuchMethodException e) { - // This version of Hadoop does not support HdfsAdmin.getEncryptionZoneForPath(). - // Hadoop 2.6.0 introduces this new method. - } - - hdfsEncryptionSupport = (m != null); - } - - return hdfsEncryptionSupport; - } - - public class HdfsEncryptionShim implements HadoopShims.HdfsEncryptionShim { - private final String HDFS_SECURITY_DEFAULT_CIPHER = "AES/CTR/NoPadding"; - - /** - * Gets information about HDFS encryption zones - */ - private HdfsAdmin hdfsAdmin = null; - - /** - * Used to compare encryption key strengths. - */ - private KeyProvider keyProvider = null; - - private final Configuration conf; - - public HdfsEncryptionShim(URI uri, Configuration conf) throws IOException { - DistributedFileSystem dfs = (DistributedFileSystem)FileSystem.get(uri, conf); - - this.conf = conf; - this.keyProvider = dfs.getClient().getKeyProvider(); - this.hdfsAdmin = new HdfsAdmin(uri, conf); - } - - @Override - public boolean isPathEncrypted(Path path) throws IOException { - Path fullPath; - if (path.isAbsolute()) { - fullPath = path; - } else { - fullPath = path.getFileSystem(conf).makeQualified(path); - } - if(!"hdfs".equalsIgnoreCase(path.toUri().getScheme())) { - return false; - } - return (hdfsAdmin.getEncryptionZoneForPath(fullPath) != null); - } - - @Override - public boolean arePathsOnSameEncryptionZone(Path path1, Path path2) throws IOException { - EncryptionZone zone1, zone2; - - zone1 = hdfsAdmin.getEncryptionZoneForPath(path1); - zone2 = hdfsAdmin.getEncryptionZoneForPath(path2); - - if (zone1 == null && zone2 == null) { - return true; - } else if (zone1 == null || zone2 == null) { - return false; - } - - return zone1.equals(zone2); - } - - @Override - public int comparePathKeyStrength(Path path1, Path path2) throws IOException { - EncryptionZone zone1, zone2; - - zone1 = hdfsAdmin.getEncryptionZoneForPath(path1); - zone2 = hdfsAdmin.getEncryptionZoneForPath(path2); - - if (zone1 == null && zone2 == null) { - return 0; - } else if (zone1 == null) { - return -1; - } else if (zone2 == null) { - return 1; - } - - return compareKeyStrength(zone1.getKeyName(), zone2.getKeyName()); - } - - @Override - public void createEncryptionZone(Path path, String keyName) throws IOException { - hdfsAdmin.createEncryptionZone(path, keyName); - } - - @Override - public void createKey(String keyName, int bitLength) - throws IOException, NoSuchAlgorithmException { - - checkKeyProvider(); - - if (keyProvider.getMetadata(keyName) == null) { - final KeyProvider.Options options = new Options(this.conf); - options.setCipher(HDFS_SECURITY_DEFAULT_CIPHER); - options.setBitLength(bitLength); - keyProvider.createKey(keyName, options); - keyProvider.flush(); - } else { - throw new IOException("key '" + keyName + "' already exists"); - } - } - - @Override - public void deleteKey(String keyName) throws IOException { - checkKeyProvider(); - - if (keyProvider.getMetadata(keyName) != null) { - keyProvider.deleteKey(keyName); - keyProvider.flush(); - } else { - throw new IOException("key '" + keyName + "' does not exist."); - } - } - - @Override - public List getKeys() throws IOException { - checkKeyProvider(); - return keyProvider.getKeys(); - } - - private void checkKeyProvider() throws IOException { - if (keyProvider == null) { - throw new IOException("HDFS security key provider is not configured on your server."); - } - } - - /** - * Compares two encryption key strengths. - * - * @param keyname1 Keyname to compare - * @param keyname2 Keyname to compare - * @return 1 if path1 is stronger; 0 if paths are equals; -1 if path1 is weaker. - * @throws IOException If an error occurred attempting to get key metadata - */ - private int compareKeyStrength(String keyname1, String keyname2) throws IOException { - KeyProvider.Metadata meta1, meta2; - - if (keyProvider == null) { - throw new IOException("HDFS security key provider is not configured on your server."); - } - - meta1 = keyProvider.getMetadata(keyname1); - meta2 = keyProvider.getMetadata(keyname2); - - if (meta1.getBitLength() < meta2.getBitLength()) { - return -1; - } else if (meta1.getBitLength() == meta2.getBitLength()) { - return 0; - } else { - return 1; - } - } - } - - @Override - public HadoopShims.HdfsEncryptionShim createHdfsEncryptionShim(FileSystem fs, Configuration conf) throws IOException { - if (isHdfsEncryptionSupported()) { - URI uri = fs.getUri(); - if ("hdfs".equals(uri.getScheme())) { - return new HdfsEncryptionShim(uri, conf); - } - } - - return new HadoopShims.NoopHdfsEncryptionShim(); - } - @Override public Path getPathWithoutSchemeAndAuthority(Path path) { return Path.getPathWithoutSchemeAndAuthority(path); diff --git a/shims/common/src/main/java/org/apache/hadoop/hive/io/HdfsUtils.java b/shims/common/src/main/java/org/apache/hadoop/hive/io/HdfsUtils.java index c90b34c..3d1bbf7 100644 --- a/shims/common/src/main/java/org/apache/hadoop/hive/io/HdfsUtils.java +++ b/shims/common/src/main/java/org/apache/hadoop/hive/io/HdfsUtils.java @@ -19,12 +19,19 @@ package org.apache.hadoop.hive.io; import java.io.IOException; +import java.net.URI; +import java.security.NoSuchAlgorithmException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.crypto.key.KeyProvider; +import org.apache.hadoop.crypto.key.KeyProvider.Options; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DistributedFileSystem; +import org.apache.hadoop.hdfs.client.HdfsAdmin; +import org.apache.hadoop.hdfs.protocol.EncryptionZone; import org.apache.hadoop.hive.shims.HadoopShims; import org.apache.hadoop.hive.shims.ShimLoader; @@ -41,7 +48,7 @@ public static long getFileId(FileSystem fileSystem, Path path) throws IOExceptio // TODO: we assume it only happens in tests. Fix? int nameHash = pathStr.hashCode(); long fileSize = fileSystem.getFileStatus(path).getLen(); - long id = ((fileSize ^ (fileSize >>> 32)) << 32) | ((long)nameHash & 0xffffffffL); + long id = ((fileSize ^ (fileSize >>> 32)) << 32) | (nameHash & 0xffffffffL); LOG.warn("Cannot get unique file ID from " + fileSystem.getClass().getSimpleName() + "; using " + id + "(" + pathStr + "," + nameHash + "," + fileSize + ")"); @@ -58,4 +65,159 @@ public static Path getFileIdPath( return (fileSystem instanceof DistributedFileSystem) ? new Path(HDFS_ID_PATH_PREFIX + fileId) : path; } + + + /** + * Compares two encryption key strengths. + * + * @param keyname1 Keyname to compare + * @param keyname2 Keyname to compare + * @return 1 if path1 is stronger; 0 if paths are equals; -1 if path1 is weaker. + * @throws IOException If an error occurred attempting to get key metadata + */ + public static int comparePathKeyStrength(Path path1, Path path2, Configuration conf) throws IOException { + EncryptionZone zone1, zone2; + + zone1 = getEZ(path1, conf); + zone2 = getEZ(path2, conf); + + if (zone1 == null && zone2 == null) { + return 0; + } else if (zone1 == null) { + return -1; + } else if (zone2 == null) { + return 1; + } + + KeyProvider keyProvider = ((DistributedFileSystem) FileSystem.get(conf)).getClient().getKeyProvider(); + KeyProvider.Metadata meta1, meta2; + + if (keyProvider == null) { + throw new IOException("HDFS security key provider is not configured on your server."); + } + + meta1 = keyProvider.getMetadata(zone1.getKeyName()); + meta2 = keyProvider.getMetadata(zone2.getKeyName()); + + if (meta1.getBitLength() < meta2.getBitLength()) { + return -1; + } else if (meta1.getBitLength() == meta2.getBitLength()) { + return 0; + } else { + return 1; + } + } + + + /** + * create encryption zone by path and keyname + * @param path HDFS path to create encryption zone + * @param keyName keyname + * @throws IOException + */ + public static void createEncryptionZone(Path path, String keyName, Configuration conf) throws IOException { + getHDFSAdmin(path, conf).createEncryptionZone(path, keyName); + } + + /** + * Checks if two HDFS paths are on the same encrypted or unencrypted zone. + * + * @param path1 Path to HDFS file system + * @param path2 Path to HDFS file system + * @return True if both paths are in the same zone; False otherwise. + * @throws IOException If an error occurred attempting to get encryption information + */ + public static boolean arePathsOnSameEncryptionZone(Path path1, Path path2, Configuration conf) throws IOException { + EncryptionZone zone1, zone2; + + zone1 = getHDFSAdmin(path1, conf).getEncryptionZoneForPath(path1); + zone2 = getHDFSAdmin(path2, conf).getEncryptionZoneForPath(path2); + + if (zone1 == null && zone2 == null) { + return true; + } else if (zone1 == null || zone2 == null) { + return false; + } + + return zone1.equals(zone2); + } + private final static String HDFS_SECURITY_DEFAULT_CIPHER = "AES/CTR/NoPadding"; + + /** + * Creates an encryption key. + * + * @param keyName Name of the key + * @param bitLength Key encryption length in bits (128 or 256). + * @throws IOException If an error occurs while creating the encryption key + * @throws NoSuchAlgorithmException If cipher algorithm is invalid. + */ + public static void createKey(String keyName, int bitLength, Configuration conf) + throws IOException, NoSuchAlgorithmException { + + KeyProvider keyProvider = ((DistributedFileSystem) FileSystem.get(conf)).getClient().getKeyProvider(); + + if (keyProvider == null) { + throw new IOException("HDFS security key provider is not configured on your server."); + } + if (keyProvider.getMetadata(keyName) == null) { + final KeyProvider.Options options = new Options(conf); + options.setCipher(HDFS_SECURITY_DEFAULT_CIPHER); + options.setBitLength(bitLength); + keyProvider.createKey(keyName, options); + keyProvider.flush(); + } else { + throw new IOException("key '" + keyName + "' already exists"); + } + } + + public static void deleteKey(String keyName, Configuration conf) throws IOException { + + KeyProvider keyProvider = ((DistributedFileSystem) FileSystem.get(conf)).getClient().getKeyProvider(); + if (keyProvider == null) { + throw new IOException("HDFS security key provider is not configured on your server."); + } + if (keyProvider.getMetadata(keyName) != null) { + keyProvider.deleteKey(keyName); + keyProvider.flush(); + } else { + throw new IOException("key '" + keyName + "' does not exist."); + } + } + + private static EncryptionZone getEZ(Path path, Configuration conf) throws IOException { + + HdfsAdmin admin = getHDFSAdmin(path, conf); + if (null == admin) { + return null; + } + + Path fullPath; + if (path.isAbsolute()) { + fullPath = path; + } else { + fullPath = path.getFileSystem(conf).makeQualified(path); + } + return admin.getEncryptionZoneForPath(fullPath); + } + + /** + * Checks if a given HDFS path is encrypted. + * + * @param path Path to HDFS file system + * @return True if it is encrypted; False otherwise. + * @throws IOException If an error occurred attempting to get encryption information + */ + + public static boolean isPathEncrypted(Path path, Configuration conf) throws IOException { + + return getEZ(path, conf) != null; + } + private static HdfsAdmin getHDFSAdmin (Path path, Configuration conf) throws IOException { + + URI uri = path.toUri(); + if(!"hdfs".equalsIgnoreCase(uri.getScheme())) { + return null; + } + return new HdfsAdmin(uri, conf); + } } diff --git a/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java b/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java index 7a5a9b5..95eff13 100644 --- a/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java +++ b/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java @@ -24,14 +24,11 @@ import java.net.URI; import java.nio.ByteBuffer; import java.security.AccessControlException; -import java.security.NoSuchAlgorithmException; import java.util.Comparator; import java.util.List; import java.util.Set; import java.util.TreeMap; -import com.google.common.annotations.VisibleForTesting; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.FSDataInputStream; @@ -585,122 +582,6 @@ public void checkFileAccess(FileSystem fs, FileStatus status, FsAction action) */ public boolean runDistCp(Path src, Path dst, Configuration conf) throws IOException; - /** - * This interface encapsulates methods used to get encryption information from - * HDFS paths. - */ - public interface HdfsEncryptionShim { - /** - * Checks if a given HDFS path is encrypted. - * - * @param path Path to HDFS file system - * @return True if it is encrypted; False otherwise. - * @throws IOException If an error occurred attempting to get encryption information - */ - public boolean isPathEncrypted(Path path) throws IOException; - - /** - * Checks if two HDFS paths are on the same encrypted or unencrypted zone. - * - * @param path1 Path to HDFS file system - * @param path2 Path to HDFS file system - * @return True if both paths are in the same zone; False otherwise. - * @throws IOException If an error occurred attempting to get encryption information - */ - public boolean arePathsOnSameEncryptionZone(Path path1, Path path2) throws IOException; - - /** - * Compares two encrypted path strengths. - * - * @param path1 HDFS path to compare. - * @param path2 HDFS path to compare. - * @return 1 if path1 is stronger; 0 if paths are equals; -1 if path1 is weaker. - * @throws IOException If an error occurred attempting to get encryption/key metadata - */ - public int comparePathKeyStrength(Path path1, Path path2) throws IOException; - - /** - * create encryption zone by path and keyname - * @param path HDFS path to create encryption zone - * @param keyName keyname - * @throws IOException - */ - @VisibleForTesting - public void createEncryptionZone(Path path, String keyName) throws IOException; - - /** - * Creates an encryption key. - * - * @param keyName Name of the key - * @param bitLength Key encryption length in bits (128 or 256). - * @throws IOException If an error occurs while creating the encryption key - * @throws NoSuchAlgorithmException If cipher algorithm is invalid. - */ - @VisibleForTesting - public void createKey(String keyName, int bitLength) - throws IOException, NoSuchAlgorithmException; - - @VisibleForTesting - public void deleteKey(String keyName) throws IOException; - - @VisibleForTesting - public List getKeys() throws IOException; - } - - /** - * This is a dummy class used when the hadoop version does not support hdfs encryption. - */ - public static class NoopHdfsEncryptionShim implements HdfsEncryptionShim { - @Override - public boolean isPathEncrypted(Path path) throws IOException { - /* not supported */ - return false; - } - - @Override - public boolean arePathsOnSameEncryptionZone(Path path1, Path path2) throws IOException { - /* not supported */ - return true; - } - - @Override - public int comparePathKeyStrength(Path path1, Path path2) throws IOException { - /* not supported */ - return 0; - } - - @Override - public void createEncryptionZone(Path path, String keyName) { - /* not supported */ - } - - @Override - public void createKey(String keyName, int bitLength) { - /* not supported */ - } - - @Override - public void deleteKey(String keyName) throws IOException { - /* not supported */ - } - - @Override - public List getKeys() throws IOException{ - /* not supported */ - return null; - } - } - - /** - * Returns a new instance of the HdfsEncryption shim. - * - * @param fs A FileSystem object to HDFS - * @param conf A Configuration object - * @return A new instance of the HdfsEncryption shim. - * @throws IOException If an error occurred while creating the instance. - */ - public HdfsEncryptionShim createHdfsEncryptionShim(FileSystem fs, Configuration conf) throws IOException; - public Path getPathWithoutSchemeAndAuthority(Path path); /** -- 1.7.12.4 (Apple Git-37)